Merge branch 'master' of http://gitlab.fangjia.com/dev0/main into open_v2

This commit is contained in:
2025-12-01 13:44:30 +08:00
23 changed files with 634 additions and 133 deletions
@@ -1,6 +1,7 @@
package com.fangjia.open.interceptors;
import com.fangjia.core.util.AnalyticsUtil;
import com.fangjia.core.util.JsonUtils;
import com.fangjia.dal.mongo.open.model.OpenResource;
import com.fangjia.dal.mongo.tool.model.OpenUriLog;
import com.fangjia.dal.sql.model.ApiUser;
@@ -12,6 +13,8 @@ import com.fangjia.open.model.BasicModel;
import com.fangjia.open.model.RestfulModelAndView;
import com.fangjia.service.open.OpenResourceService;
import com.fangjia.service.open.OpenUriLogService;
import com.fasterxml.jackson.databind.ObjectMapper;
import net.sf.json.JSONNull;
import net.sf.json.JSONObject;
import net.sf.json.JsonConfig;
import net.sf.json.processors.JsonValueProcessor;
@@ -22,17 +25,13 @@ import org.sothis.web.mvc.ActionContext;
import org.sothis.web.mvc.ActionInvocation;
import org.sothis.web.mvc.ActionInvocationException;
import org.sothis.web.mvc.Interceptor;
import javax.servlet.http.HttpServletRequest;
import java.math.BigDecimal;
import java.text.SimpleDateFormat;
import java.util.Collection;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.*;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
/**
* API审计日志监控器
*
@@ -116,7 +115,6 @@ public class AuditInterceptor implements Interceptor {
}
return params;
}
/**
* @param request
* @param apiUser
@@ -163,7 +161,7 @@ public class AuditInterceptor implements Interceptor {
if(isSaveResult(request)){
JSONObject json = JSONObject.fromObject(result, DEFAULT_JSON_CONFIG);
json.remove("tid");
openUriLog.setResMap(json);
openUriLog.setResMap((Map<String, Object>)JsonUtils.convertJsonToJavaObj(json, Map.class));
}
} catch (Exception e) {
LOGGER.error("", e);
@@ -813,8 +813,36 @@ class CommonDatautils {
// q.put("status",new BasicDBObject('$ne', 0))
q.put("cat", cat);
q.put("city", city)
// q.put("loop_line_info",new BasicDBObject('$exists',false))
// q.put("name","经纬天地园二期")
// q.put("name","扬州北街8号")
DBObject keys = new BasicDBObject();
if(fields){
for(def field : fields){
keys.put(field, 1);
}
}
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(5000)
def rets = []
for(def d : dbsour){
rets << d
}
return rets;
}
public static def findNewUpdateDistricts(def city, def cat, def fields){
def startDate = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd")
DBObject q = new BasicDBObject();
q.put("status",0)
// q.put("status",new BasicDBObject('$ne', 0))
q.put("cat", cat);
q.put("city", city)
q.put("m_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)))
// q.put("loop_line_info",new BasicDBObject('$exists',false))
// q.put("name","扬州北街8号")
DBObject keys = new BasicDBObject();
if(fields){
@@ -5817,4 +5845,68 @@ class CommonDatautils {
}
return address;
}
static def findSeawater(city, lng, lat, distance, p_date, cat){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:goojia123456".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.206:9200/seawater-*/_search"
def body = """
{
"query": {
"bool": {
"must": [
{
"geo_distance": {
"distance": "${distance}",
"location": {
"lat": ${lat},
"lon": ${lng}
}
}
},
{
"term": {
"city": "${city}"
}
},
{
"term": {
"cat": "${cat}"
}
},
{
"range": {
"publish_date": {
"gte": "${p_date}",
"format": "yyyy-MM-dd"
}
}
}
]
}
},
"sort": [
{
"publish_date": {
"order": "desc"
}
}
],
"size":30
}
"""
HttpRequest request = new HttpRequest()
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = new JsonSlurper().parseText(res)
def esObj = null
if(response && response.hits){
esObj = response.hits.hits
}
return esObj
}
}
@@ -211,6 +211,7 @@ class DistrictInfoExportBatch extends BaseBatch{
def getFzDistricts(def city, def mdate, def endMDate){
def q = new BasicDBObject()
q.put("city", city)
if(mdate){
q.put("m_date", new BasicDBObject('$gte', mdate).append('$lt', endMDate))
}
@@ -99,6 +99,7 @@ class DistrictPriceExportBatch extends BaseBatch{
def getFzDistricts(def city, def mdate, def endMDate){
def q = new BasicDBObject()
q.put("city", city)
if(mdate){
q.put("m_date", new BasicDBObject('$gte', mdate).append('$lt', endMDate))
}
@@ -37,14 +37,14 @@ class DistrictSellStatBatch extends BaseBatch {
@Override
void process(Object args) {
// def months = DateUtils.getMonthList(202206)
// months = months[0..(months.size()-2)]
// println months
// for(def month : months){
// def curDate = DateUtils.parseFromTightByPattern(month.toString()+"01", "yyyyMMdd")
// calcMonthlyPrice(curDate)
// }
calcMonthlyPrice()
def months = DateUtils.getMonthList(202510)
months = months[0..(months.size()-1)]
println months
for(def month : months){
def curDate = DateUtils.parseFromTightByPattern(month.toString()+"01", "yyyyMMdd")
calcMonthlyPrice(curDate)
}
// calcMonthlyPrice()
}
@@ -69,7 +69,9 @@ class DistrictSellStatBatch extends BaseBatch {
if(!curDate){
curDate = new Date()
}
def citys = CommonUtils.findAllCity()
def citys = ["上海"]//CommonUtils.findAllCity()
// citys.remove("上海")
// citys.remove("福州")
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1).append("status", 1)
def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(curDate))
@@ -87,22 +89,23 @@ class DistrictSellStatBatch extends BaseBatch {
for(def city : citys){
index++
def districts = []
if(city == "福州"){
if(day == 2){
if(day == 29){
districts = FZ_DIST_COLL.find(new BasicDBObject("city", city), keys).toList()
}else{
districts = FZ_DIST_COLL.find(new BasicDBObject("city", city).append("m_date", new BasicDBObject('$gte',startMDate).append('$lt', endMDate)), keys).toList()
}
}else{
def q = new BasicDBObject("city", city).append("cat", "district").append("status", 0)
// q.put("name", "翠园楼")
if(day == 2){
// q.put("name", "越华路小区")
// if(day == 30){
districts = SEAWEED_MONGO_COLL.find(q, keys).batchSize(2000).toList()
}else{
q.put("m_date", new BasicDBObject('$gte',startMDate).append('$lt', endMDate))
districts = SEAWEED_MONGO_COLL.find(q, keys).batchSize(2000)
}
// }else{
// q.put("m_date", new BasicDBObject('$gte',startMDate).append('$lt', endMDate))
// districts = SEAWEED_MONGO_COLL.find(q, keys).batchSize(2000)
// }
}
println index+" "+city +" "+districts.size()+" "+ curDate.toString()
@@ -155,7 +158,7 @@ class DistrictSellStatBatch extends BaseBatch {
def key = dist.region +"##" + dist.name
def weekendPriceObj = weekendPriceMap.get(key)
if(weekendPriceObj){
if(weekendPriceObj && weekendPriceObj.avg_price){
def curPriceObj = curMonthlyPriceMap.get(key)
def lastMonthPriceObj = lastMonthlyPriceMap.get(key)
def lastYearPriceObj = lastYearPriceMap.get(key)
@@ -216,8 +219,19 @@ class DistrictSellStatBatch extends BaseBatch {
data.put("credit", weekendPriceObj.credit)
def q = new BasicDBObject()
q.put("date", curMonth)
q.put("commid", dist._id.toString())
if(curPriceObj.commid){
q.put("date", curMonth)
q.put("commid", dist._id.toString())
}else{
q.put("date", curMonth)
q.put("city", dist.city)
q.put("region", dist.region)
q.put("district_name", dist.name)
data.put("commid", dist._id.toString())
}
DISTRICT_SELL_STAT.update(q , new BasicDBObject('$set', data),true,false)
// DISTRICT_SELL_STAT.update(new BasicDBObject("_id", curPriceObj._id) , new BasicDBObject('$set', data),true,false)
println curMonth+" [update]"+totalCount.get()+" "+ dist.city+" "+dist.name
@@ -239,6 +253,8 @@ class DistrictSellStatBatch extends BaseBatch {
DISTRICT_SELL_STAT.save(data)
println curMonth+" [save]"+totalCount.get()+" "+ dist.city+" "+dist.name
}
}
} catch (Exception e) {
@@ -281,6 +297,10 @@ class DistrictSellStatBatch extends BaseBatch {
if(d.avg_price_2b && d.value_type!=1){
d.avg_price = d.avg_price_2b
}
}else{
if(d.avg_price_2b){
d.avg_price = d.avg_price_2b
}
}
priceMap.put(d.region +"##"+d.district_name, d)
}
@@ -831,7 +831,7 @@ class OpenApiUtils {
static def addrVerifyGuangda(city, address){
address = URLEncoder.encode(address, "utf-8");
def url = "http://192.168.0.100:32080/FangJiaAddressService/cebEnterpriseVillageIdentify?address=${address}"
// def url = "http://localhost:8080/fangjia-open/addrSvc/njcbVillageIdentify?address=${address}&type=${type}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
// def url = "http://localhost:8080/FangJiaAddressService/cebEnterpriseVillageIdentify?address=${address}"
if(city){
url = url + "&city=${city}"
}
@@ -847,23 +847,6 @@ class OpenApiUtils {
}
static def addrVerifyShBank(city, address){
address = URLEncoder.encode(address, "utf-8");
// def url = "http://192.168.0.100:32080/FangJiaAddressService/cebEnterpriseVillageIdentify?address=${address}"
def url = "http://localhost:8080/addrSvc/boscEnterpriseVillageIdentify?address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
if(city){
url = url + "&city=${city}"
}
String content = new HttpRequest().get(url, "utf-8");
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==200 && doc.result){
def dtos = doc.result
return dtos
}
}
return null;
}
static def addrVerifyBase(city, address){
@@ -945,4 +928,17 @@ class OpenApiUtils {
return null;
}
static def lactionEvaluation(area, lng, lat){
def url = "http://192.168.0.100:30020/avmWeb/avmAnn?area=${area}&location=lng"+","+lat
String content = new HttpRequest().get(url, "utf-8");
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.msgCode ==200 && doc.result){
def dtos = doc.result
return dtos
}
}
return null;
}
}
@@ -36,7 +36,8 @@ class SuzouPocTest {
// task.newRun()
// task.test()//有楼层
// task.testEval()//无楼层
task.level()//楼盘评级
// task.level()//楼盘评级
task.house()
}
def test(){
@@ -97,36 +98,73 @@ class SuzouPocTest {
}
def testEval(){
ExcelUtils.modify("D:\\数据\\其它\\附件1:POC测试样本(匹配).xlsx", { d ->
if(d['O'] && !d['T']){
ExcelUtils.modify("D:\\数据\\宏信息\\样本房源-估值.xlsx", { d ->
if(!d['O'] && d['E']){
// def city = d['C']
// if(city == "市辖区"){
// city = d['B']
// }
def obj = CommonDatautils.findDistrictById(d['O'])
if(obj){
d['R'] = obj.city
d['S'] = obj.region
d['T'] = obj.name
def ret1 = OpenApiClient.addressEvaluation(obj.city, obj.region, obj.city+obj.region+obj.name, obj.name, d['F'], null, null)
// def obj = CommonDatautils.findDistrictById(d['O'])
// if(obj){
// d['R'] = obj.city
// d['S'] = obj.region
// d['T'] = obj.name
def ret1 = OpenApiClient.addressEvaluation(d['C'], d['D'], d['C']+d['D']+d['E'], d['E'], d['I'], null, null)
if (ret1) {
d['P'] = ret1.totalPrice
d['Q'] = ret1.avgPrice
}else{
def dbo = CommonDatautils.queryFirstDistrictPrice(obj.city, obj.region, obj.name)
if(dbo) {
d['P'] = (int)(dbo.avg_price_2b*d['F'].toString().toDouble())
d['Q'] = dbo.avg_price_2b
}
d['J'] = ret1.totalPrice
d['K'] = ret1.avgPrice
d['L'] = ret1.city
d['M'] = ret1.district
d['N'] = ret1.name
}
}
// else{
// def dbo = CommonDatautils.queryFirstDistrictPrice(obj.city, obj.region, obj.name)
// if(dbo) {
// d['P'] = (int)(dbo.avg_price_2b*d['F'].toString().toDouble())
// d['Q'] = dbo.avg_price_2b
//
// }
//
// }
// }
}
});
}
def house(){
def cases = []
cases << ["城市", "区域","板块", "名称", "面积", "单价","总价","装修", "朝向", "距离","所在楼层","总楼层","室","厅","卫","发布时间","标题", "序号"]
ExcelUtils.modify("D:\\数据\\宏信息\\样本房源-估值.xlsx", { d ->
if(d['L']){
def obj = CommonDatautils.findDistrict(d['L'], d['M'], d['N'], "district")
if(obj){
def lng = obj.lng/10000000000.0
def lat = obj.lat/10000000000.0
def location = String.valueOf(lng)+","+ String.valueOf(lat)
def result = OpenApiClient.houseAroundBylocationAction(location, d['I'], 0, 0, 50)
if(result){
for(def h : result){
cases << [h.city, h.district, h.block, h.name, h.size, h.unitPrice, h.totalPrice, h.decoration, h.direction, h.distance, h.floor,h.maxFloor, h.room, h.hall, h.toilet, h.publishDate, h.title, d["A"],"挂牌"]
}
}
def address = d['L']+d['M']+d['N']
def dealRets = OpenApiClient.dealHouse(d['L'], d['M'], d['N'], address, d['I'])
if(dealRets){
for(def h : dealRets){
cases << [h.city, h.district, "", h.name, h.size, h.avgPrice, h.totalPrice, h.decoration, h.direction, "", h.floor,h.maxFloor, h.room, h.hall, h.toilet, h.tradeDate, "", d["A"], "成交"]
}
}
}
}
})
println cases.size()
ExcelUtils.write("D:\\数据\\宏信息\\样本房源-估值.xlsx", cases, "样本")
}
def newRun(){
println("start:"+new Date().toString())
@@ -38,8 +38,6 @@ class DevUpdateTestBatch {
//
// task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是")
}
def addDev(path){
@@ -191,7 +191,7 @@ class NewGaoDePoiBatch extends BaseBatch{
if(smallArr && smallArr.size()>=1){
saveObj.small_type = smallArr[0]
}else{
saveObj.small_type = smallstr
saveObj.8 = smallstr
}
}else{
saveObj.small_type = saveObj.secondary_type
@@ -57,9 +57,9 @@ class TownStandardBatch extends BaseBatch{
}
def TownStandard(){
townStandardByCat("office")
// townStandardByCat("office")
// townStandardByPoi()
// addTownByTownShape()//为没有乡镇街道的poi,补充乡镇街道,忽略区域,并使用地图
addTownByTownShape()//为没有乡镇街道的poi,补充乡镇街道,忽略区域,并使用地图
// standardTown()
// updateTownData()
@@ -0,0 +1,165 @@
package com.fangjia.script.gj.district
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBList
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONNull
import org.bson.types.ObjectId
class DistrictLocationGeo {
final DBCollection seaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "seaweed_gaode_pois", "192.168.0.235", 27018)
final DBCollection allSeaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "all_seaweed_gaode_pois", "192.168.0.235", 27018)
static main(args) {
def task = new DistrictLocationGeo()
task.run()//通过经纬度,调用高德逆解析,并保存数据
// task.run2();//通过地址,调用高德逆解析
}
def run(){
ExcelUtils.modify("D:\\数据\\京东\\失败.xlsx",{ d->
if(d['E']){
def jsonObject = GaoDeApiUtils.geocode(d['E'])
if(jsonObject){
d["F"] = "高德"
def dbo = new BasicDBObject()
dbo.put("old_id", d['A'])
dbo.put("city", d['B'])
dbo.put("region", d['C'])
dbo.put("name", d['D'])
dbo.put("old_location", d['E'])
dbo.put("formatted_address", jsonObject.regeocode.formatted_address)
def addrComp = jsonObject.regeocode.addressComponent
// addrComp.remove("neighborhood")
// addrComp.remove("building")
dbo.put("addressComponent", addrComp)
dbo.put("pois", jsonObject.regeocode.pois)
dbo.put("roads", jsonObject.regeocode.roads)
dbo.put("roadinters", jsonObject.regeocode.roadinters)
dbo.put("aois", jsonObject.regeocode.aois)
dbo = cleanGeoData(dbo)
dbo = convertToDBObject(dbo)
println(dbo)
allSeaweedGaodePoisColl.save(dbo)
}
}
})
}
static BasicDBObject convertToDBObject(Map map) {
if (!map) return null
BasicDBObject dbObject = new BasicDBObject()
map.each { key, value ->
def convertedValue = convertValue(value)
if (convertedValue != null) {
dbObject.put(key, convertedValue)
}
}
return dbObject
}
static Object convertValue(Object value) {
if (value == null) return null
switch (value) {
case Map:
return convertToDBObject(value as Map)
case List:
return convertList(value as List)
default:
return value
}
}
static BasicDBList convertList(List list) {
BasicDBList dbList = new BasicDBList()
list.each { item ->
def convertedItem = convertValue(item)
if (convertedItem != null) {
dbList.add(convertedItem)
}
}
return dbList
}
static def cleanGeoData(data) {
if (data == null) return null
def result = [:]
data.each { key, value ->
def cleanedValue = cleanValue(value)
if (cleanedValue != null && !isEmptyCollection(cleanedValue)) {
result[key] = cleanedValue
}
}
return result
}
static def cleanValue(value) {
if (value == null) return null
switch (value) {
case Map:
def cleanedMap = [:]
value.each { k, v ->
def cleaned = cleanValue(v)
if (cleaned != null && !isEmptyCollection(cleaned)) {
cleanedMap[k] = cleaned
}
}
return cleanedMap ? cleanedMap : null
case List:
def cleanedList = value.collect { cleanValue(it) }
.findAll { it != null && !isEmptyCollection(it) }
return cleanedList ? cleanedList : null
case JSONNull:
return null
default:
// 检查是否是包含JSONNull的字符串表示
if (value instanceof String && value.contains("JSONNull")) {
return null
}
return value
}
}
static def isEmptyCollection(obj) {
if (obj instanceof Map) return obj.isEmpty()
if (obj instanceof List) return obj.isEmpty()
return false
}
def run2(){
ExcelUtils.modify("D:\\数据\\昆仑\\263未出居委数据.xlsx",{ d->
def jsonObject = GaoDeApiUtils.geocodeByAddress(d['C'], d['B'])
Thread.sleep(1000)
def count = 0
if (jsonObject) {
def rets = jsonObject.geocodes
if(rets){
def dbo = rets[0]
if(dbo){
d['D'] = dbo.formatted_address
d['E'] = dbo.level
}
}
count = jsonObject.regeocode.pois.size()
}
d['D'] = count
})
}
}
@@ -355,7 +355,7 @@ class FangjiaClientUtils {
address = address.toString().replace("{","")
int retryCount = 0
while (retryCount < 3) {
url = "http://192.168.0.93:8080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true&fields=dev_confi"
url = "http://192.168.0.100:32080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true&fields=dev_confi"
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
@@ -6,17 +6,15 @@ import com.fangjia.script.Watcher
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.google.common.hash.*
import com.google.common.hash.BloomFilter
import com.google.common.hash.Funnels
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.sql.Sql
import org.apache.commons.lang.StringUtils
import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
@@ -24,15 +22,12 @@ import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.nio.charset.StandardCharsets
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher
import java.util.regex.Pattern
/**
* 建行模型估值项目
* 出售房源数据同步
@@ -201,7 +196,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{
} finally {
latch.countDown()
}
};
};
})
}
latch.await()
@@ -492,8 +487,8 @@ class FjwBigdataForsaleBatch extends BaseBatch{
def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){
// def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
def date = DateUtils.addMonth(endDate.toString(), -12, "yyyyMMdd")
def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
// def date = DateUtils.addMonth(endDate.toString(), -12, "yyyyMMdd")
def startMdate = DateUtils.parseFromTightByPattern(date,"yyyyMMdd")//DateUtils.parseFromTightByPattern("20240519","yyyyMMdd")
def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")//DateUtils.parseFromTightByPattern("20240718", "yyyyMMdd")
@@ -1,19 +1,36 @@
package com.fangjia.script.gj.village
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.ExcelUtils
class KunlunVillage {
static main(args) {
def task = new KunlunVillage();
task.run()
// task.addrCheck()
}
def run(){
ExcelUtils.multiThreadingModify("D:\\数据\\昆仑\\3987户需批量更新涉农信息明细-20251023-结果.xlsx",{ d->
def addrCheck(){
if(!d['R'] && d['B']){
def dto = OpenApiUtils.addrVerifyGuangda(null, d['B'])
ExcelUtils.modify("D:\\数据\\昆仑\\2025年11月18日批次跑批涉农地址-发估家.xlsx",{ d->
def obj = FangjiaClientUtils.formatNinelevel(null, d["C"])
if(obj){
println obj
d['AB'] = obj.addressAssessment.msg
}
})
}
def run(){
def index = 0
ExcelUtils.modify("D:\\数据\\昆仑\\2025年11月18日批次跑批涉农地址-发估家.xlsx",{ d->
index++
if(!d['R'] && d['C']){
println index
def dto = OpenApiUtils.addrVerifyBase(null, d['C'])
if(dto){
// def dto = jsonObj.result
d['F'] = dto.province
@@ -10,7 +10,7 @@ class XiamenVillage {
}
def run(){
ExcelUtils.multiThreadingModify("D:\\数据\\厦门\\涉农组织类型判定(to估家,2025-09-30)-结果.xlsx",{ d->
ExcelUtils.multiThreadingModify("D:\\数据\\厦门\\涉农组织类型判定(to估家,2025-10-31)-以此为准.xlsx",{ d->
if(!d['R']){
def dto = OpenApiUtils.addrVerifyXiameng(null, d['B'], 2)
@@ -1,21 +1,16 @@
package com.fangjia.script.house
import com.fangjia.core.util.DateUtil
import com.fangjia.datastore.driver.DSCollection
import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
@@ -48,14 +43,17 @@ class DistrictSellHouseStatBatch extends BaseBatch{
for(def city : citys){
index ++ ;
def districts = CommonDatautils.getAllDistrictBaseInfo(city, "district")
def sellRet = houseStatCount(districts, startDate, endDate, sDate);
String start = DateUtils.formatDateByPattern(startDate, "yyyyMMdd")
def sellRet = houseStatCount(districts, start.toInteger(), sDate);
saveResult(city, sDate, sellRet);
println "[${city} ${index}/${citys.size()}]" +new Date().toString();
}
}
def houseStatCount(def districts, def startDate, def endDate, def sDate){
def houseStatCount(def districts, def startDate, def sDate){
def index = 0;
@@ -75,8 +73,12 @@ class DistrictSellHouseStatBatch extends BaseBatch{
void run() {
index++;
def map = [:];
map.sell_count = findHouseCount(dist, "sell", startDate, endDate);
map.rent_count = findHouseCount(dist, "rent", startDate, endDate);
// map.sell_count = findHouseCount(dist, "sell", startDate, endDate);
// map.rent_count = findHouseCount(dist, "rent", startDate, endDate);
map.sell_count = CommonDatautils.findHouseCountByDate(dist.city, dist.region, dist.name, dist.alias, "sell", startDate)
map.rent_count = CommonDatautils.findHouseCountByDate(dist.city, dist.region, dist.name, dist.alias, "rent", startDate)
map.city = dist.city;
map.region = dist.region;
map.name = dist.name;
@@ -1,5 +1,6 @@
package com.fangjia.script.jiangnanblank
import com.fangjia.core.util.DateUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
@@ -7,24 +8,17 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.rabbitmq.client.AMQP
import net.sf.json.JSONObject
import org.apache.poi.hssf.usermodel.HSSFWorkbook
import org.bson.types.ObjectId
import java.text.DecimalFormat
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher
import java.util.regex.Pattern
/**
* Created by gaojuan on 2017/12/11.
* 批量全国小区及商铺字段转换
@@ -33,7 +27,7 @@ import java.util.regex.Pattern
class CZDataBatch extends BaseBatch{
public static def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
def fields = ["city","region","name", "address", "cat","lat2","lng2","alias","property_life","estate_type2","build_date","build_structure","build_type","finish_building_date","live_date","open_date","virescence_percent","cubage_percent","estate_company","estate_charge","car_site", "build_type"]
def fields = ["city","region","name", "address", "cat","lat2","lng2","lat","lng","alias","property_life","estate_type2","build_date","build_structure","build_type","finish_building_date","live_date","open_date","virescence_percent","cubage_percent","estate_company","estate_charge","car_site", "build_type"]
public static DBCollection DIST_COLL = MongoUtils.getCollectionByName2("business", "jn_bank_district", dbUrl)
//final def czIdMap = readCZDistrict("d:\\常州数据结果\\原始数据\\常州江南银行住宅编号.xls");
@@ -47,6 +41,8 @@ class CZDataBatch extends BaseBatch{
@Override
void process(Object args) {
println "args ²ÎÊý£º"+args.toString()
def citys = CommonDatautils.findAllCity();
citys.remove("常州");
@@ -56,13 +52,22 @@ class CZDataBatch extends BaseBatch{
def weekend = DateUtils.getFirstDayOfWeek(new Date())
def statMap = CommonDatautils.queryAllDistrictPriceByWeekend(city, weekend)
def commids = findAllJnBankDatas(city)
def districts = converList(CommonDatautils.findAllDistricts(city, "district", fields));
FzDataUtils.removeNoUpdate(city, districts)
//
def districts = null;
def shops = null;
if(args && args[0]== "all"){
districts = converList(CommonDatautils.findAllDistricts(city, "district", fields));
FzDataUtils.removeNoUpdate(city, districts)
}else{
districts = converList(CommonDatautils.findNewUpdateDistricts(city, "district", fields));
}
districtRun(districts, statMap, commids);
def shops = CommonDatautils.findAllDistricts(city, "shop", fields);
if(args && args[0]== "all"){
shops = CommonDatautils.findAllDistricts(city, "shop", fields);
}else{
shops = CommonDatautils.findNewUpdateDistricts(city, "shop", fields);
}
shops = mergeShop(shops);
shopRun(shops);
}
@@ -219,19 +224,22 @@ class CZDataBatch extends BaseBatch{
// 商铺名称
map.name = shop.name
map.alias = shop.alias
map.build_date = getBuildDate(shop);
map.build_date = FzDataUtils.getBuildDate(shop);
//附属属性
map.build_type = shop.build_type
map.property_life = shop.property_life
map.estate_type2 = shop.estate_type2
map.build_structure = shop.build_structure
// 楼盘均价
map.avg_price = getShopPrice(priceString, shop.city, shop.region, shop.name);
map.avg_price = FzDataUtils.getShopPrice(priceString, shop.city, shop.region, shop.name);
if(!map.avg_price){
map.avg_price = getSellPrice(shop.city, shop.lng/10000000000.0, shop.lat/10000000000.0)
}
// 省 市 区
if(shop.city == "常州"){
map.province_code = regionMap.get("˝­ËŐ");
map.city_code = regionMap.get(shop.city);
map.region_code = regionMap.get(shop.region);
map.province_code = FzDataUtils.regionMap.get("½­ËÕ");
map.city_code = FzDataUtils.regionMap.get(shop.city);
map.region_code = FzDataUtils.regionMap.get(shop.region);
map.province = "江苏";
map.price_old_string = priceString;
map.heights = heights;
@@ -241,32 +249,73 @@ class CZDataBatch extends BaseBatch{
// 地址
map.address = shop.address
// 所属区域
map.in_region_code = getRegionCode(shop.region);
map.in_region_code = FzDataUtils.getRegionCode(shop.region);
// 商铺类型
map.estate_type = getShopEstateTypeCode(shop.estate_type2);
map.estate_type = FzDataUtils.getShopEstateTypeCode(shop.estate_type2);
// 公共服务配套
map.public_service_set_code = getPublicServiceSetCode(shop.city, shop.lat2, shop.lng2);
map.public_service_set_code = FzDataUtils.getPublicServiceSetCode(shop.city, shop.lat2, shop.lng2);
// 基础设施
map.facilities_code = "01";
// 商业密度
map.sophistication_code = getSophisticationCode(shop.city, shop.lat2, shop.lng2);
map.sophistication_code = FzDataUtils.getSophisticationCode(shop.city, shop.lat2, shop.lng2);
// 交通条件
map.traffic_code = getTrafficCode(shop.city, shop.lat2, shop.lng2);
map.traffic_code = FzDataUtils.getTrafficCode(shop.city, shop.lat2, shop.lng2);
// 泊车条件 /
map.stop_car_code = "01";
// 临街道路
map.street_code = getStreetCode(street);
map.street_code = FzDataUtils.getStreetCode(street);
// 租金
map.shop_rent = getRentNumber(rentPrice, shop.city, shop.region, shop.name);
map.shop_rent = FzDataUtils.getRentNumber(rentPrice, shop.city, shop.region, shop.name);
if(!map.shop_rent){
map.shop_rent = getRentPrice(shop.city, shop.lng/10000000000.0, shop.lat/10000000000.0)
}
// 物业公司
map.estate_company = getEstateCompany(shop.estate_company);
map.estate_company = FzDataUtils.getEstateCompany(shop.estate_company);
// 物业费(每月每平米)
map.estate_charge = getEstateCharge(shop.estate_charge);
map.estate_charge = FzDataUtils.getEstateCharge(shop.estate_charge);
saveShop(map);
}
def getSellPrice(def city, def lng, def lat){
def p_date = DateUtils.formatDateByPattern(DateUtil.add(new Date(), -365),'yyyy-MM-dd')
def rets = CommonDatautils.findSeawater(city, lng, lat, "1km", p_date, "shop_sell")
if(rets){
def prices = []
for(def r : rets){
prices << r._source.avg_price
}
if(prices){
def price = (int)(prices.sum()/prices.size())
return price
}
}
return null;
}
def getRentPrice(def city, def lng, def lat){
def p_date = DateUtils.formatDateByPattern(DateUtil.add(new Date(), -365),'yyyy-MM-dd')
def rets = CommonDatautils.findSeawater(city, lng, lat, "1km", p_date, "shop_rent")
if(!rets){
rets = CommonDatautils.findSeawater(city, lng, lat, "2km", p_date, "shop_rent")
if(!rets){
rets = CommonDatautils.findSeawater(city, lng, lat, "3km", p_date, "shop_rent")
}
}
if(rets){
def prices = []
for(def r : rets){
prices << r._source.avg_price
}
if(prices){
def day_price = (prices.sum()/prices.size())/30
return DataHandleUtils.getDoubleRound(2, day_price)
}
}
return null;
}
def saveShop(def m){
DBObject q = new BasicDBObject();
q.put("city",m.city)
@@ -809,4 +809,5 @@ class FzDataUtils {
}
}
@@ -20,10 +20,12 @@ class TestEvaluation {
// task.match()
// task.houseSimilar()
task.newMatch()
// task.newMatch()
// task.houseSimilar()
task.newAndOldMatch()
// task.newAndOldMatch()
task.addFjwToBprice("D:\\数据\\估值\\地址估价-匹配(加写字楼-商铺) - v2.xlsx")
}
def houseSimilar(){
@@ -206,4 +208,19 @@ class TestEvaluation {
})
}
def addFjwToBprice(path){
ExcelUtils.multiThreadingModify(path,{d->
def dist = CommonDatautils.findDistrictById(d['B'])
if(dist){
def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
if(dbo){
if(dbo.avg_price_2b){
d['H'] = dbo.avg_price_2b
}
}
}
})
}
}
@@ -414,7 +414,7 @@ public class ExcelUtils{
def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
def body = data.subList(1, data.size())
def nThread = Runtime.getRuntime().availableProcessors() * 6
def nThread = Runtime.getRuntime().availableProcessors() * 8
println nThread
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
final AtomicInteger counter = new AtomicInteger(0)
@@ -8,6 +8,7 @@ import com.mongodb.DBCollection
import com.mongodb.DBCursor
import net.sf.json.JSONArray
import net.sf.json.JSONObject
import net.sf.json.groovy.JsonSlurper
import org.apache.commons.lang.StringUtils
import org.apache.commons.lang.SystemUtils
@@ -1577,5 +1578,73 @@ public class OpenApiClient {
public static def houseAroundBylocationAction(location, size, propertyType, houseType, pageSize) {
def url ="https://open.fangjia.com/house/around?location="+location+"&size="+size+"&pageSize="+pageSize+"&propertyType="+propertyType+"&houseType="+houseType
url = url +"&token="+TOKEN
def content = new HttpRequest().newGet(url, null, "utf-8")
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==200 && doc.result){
def dtos = doc.result
return dtos
}
}
return null
}
static def dealHouse(city, region, name, area, address){
def url ="https://open.fangjia.com/property/deal?city="+city+"&address="+address+"&district="+region+"&name="+name+"&size="+area
url = url +"&token="+TOKEN
def content = new HttpRequest().newGet(url, null, "utf-8")
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==200 && doc.result){
def dtos = doc.result
return dtos
}
}
return null
}
public static JSONObject propertyHouseAction(city, district, name, size, propertyType) {
try {
Map<String, String> params = new HashMap<String, String>();
params.put("city", city);
params.put("district", district)
params.put("name", name)
params.put("size", size.toString());
params.put("token", TOKEN);
params.put("propertyType", propertyType)
params.put("__refresh_cache","1")
String content = new HttpRequest().get("https://open.fangjia.com/property/house", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
while (result == null) {
result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/property/house", params, "utf-8")
println("${city} ${name} ³ÌÐòµ÷ÓÃÒì³£")
Thread.sleep(6000)
}
if (result.getInt("code") == 0) {
return result.getJSONObject("result");
} else if (result.getInt("code") !=0) {
println("${city} ${name} ")
}
} catch (Exception e) {
println("${city} ${name} ${e.getMessage()}")
e.printStackTrace();
}
return null
}
}
@@ -1,19 +1,17 @@
package com.fangjia.script.yinjihaun.amap
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.thirdPriceCrawl.UserAgents
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.JacksonUtil
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.jsoup.Jsoup
import java.util.zip.GZIPInputStream
class GaoDeApiUtils {
//94c74c60e102972a8d17b7b75d7403be
def static GAO_DE_AK = "94c74c60e102972a8d17b7b75d7403be" //±¦½£:c6fcd4bfbc40d73ee762b8115dca3390
//67423021cae3b58e5c2ac595a389b036 :临时买的key
def static GAO_DE_AK = "67423021cae3b58e5c2ac595a389b036" //宝剑:c6fcd4bfbc40d73ee762b8115dca3390
//15021384995£ºdfc2245bac8a7b2d5b62e060e59d4eca
/**
@@ -29,6 +27,11 @@ class GaoDeApiUtils {
params.put("key", GAO_DE_AK)
params.put("location", location)
params.put("extensions","all")
params.put("radius", "3000")
// params.put("poitype", "120000|120200|120203|120300|120301|120302|120303")//纯住宅类型
//京东测试允许类型
params.put("poitype", "100000|100100|100101|100102|100103|100104|100105|100200|100201|120000|120200|120203|120300|120301|120302|120303|120304|190000|190100|190400|190401|190402|190403|190500|190600|190700")
String content
def stop = true
JSONObject jsonObj = null
@@ -37,6 +40,7 @@ class GaoDeApiUtils {
try {
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8")
} catch (Exception e) {
println location + " 出错:"+content
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8")
}
jsonObj = JSONObject.fromObject(content)
@@ -55,6 +59,40 @@ class GaoDeApiUtils {
return jsonObj
}
def static geocodeByAddress(String city, String address) {
Map<String, String> params = new HashMap<String, String>()
params.put("key", GAO_DE_AK)
params.put("address", address)
params.put("batch", "true")
params.put("city", city)
// params.put("poitype", "120000|120200|120203|120300|120301|120302|120303")
String content
def stop = true
JSONObject jsonObj = null
def errorCount = 1
while (stop){
try {
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/geo", params, "utf-8")
} catch (Exception e) {
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/geo", params, "utf-8")
}
jsonObj = JSONObject.fromObject(content)
println jsonObj
if(jsonObj.status!="1"){
println jsonObj
errorCount ++
Thread.sleep(3000)
if(errorCount>3){
stop = false
}
}else{
stop = false
}
}
return jsonObj
}
def static geo(city, address) {
Map<String, String> params = new HashMap<String, String>()
params.put("key", "a34894d151fffff94f9a33bae3d5565c")//ÐíéF½Ü
@@ -339,9 +339,13 @@ public class AddressParserServiceImpl implements AddressParserService {
}
}
}
district = districtService.findById(district.getId(), MATCH_PROPERTY_FIELDS);
if(district == null){
logger.error("未找到匹配的CPDB数据:" + addr + "ID:" + district.getId());
logger.error("未找到匹配的CPDB数据:" + addr + "ID:" + matchedMate.getId());
}else{
district = districtService.findById(district.getId(), MATCH_PROPERTY_FIELDS);
}
if(district == null){
logger.error("未找到匹配的CPDB数据:" + addr + "ID:" + matchedMate.getId());
}
return new Building(district, scoreResult.getScore(matchedMate));
}else{