From 5033b0b4753fda8669e9e6c0795b85afce5fdff7 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Thu, 4 Dec 2025 11:07:29 +0800 Subject: [PATCH 1/2] update --- .../script/ccbevaluation/AddSzDealData.groovy | 2 +- .../ccbevaluation/CcbCommonUtils.groovy | 2 +- .../gj/district/DistrictLocationGeo.groovy | 53 +++---- .../script/gj/district/JindDongGeoTest.groovy | 144 ++++++++++++++++++ 4 files changed, 171 insertions(+), 30 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/JindDongGeoTest.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy index 9ebbbcaa2..0f76938ce 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy @@ -9,7 +9,7 @@ import com.mongodb.DBCollection * 深圳公众号成交数据录入 */ -def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳9月份成交数据.xlsx", 0, 1) +def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳10月份成交数据11.20.xlsx", 0, 1) def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 4dd815364..88194c547 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -443,7 +443,7 @@ class CcbCommonUtils { } // q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13 -// q.put("name","龙源星城") +// q.put("name","登高西路安置小区C区") // q.put("name_cat", 0); // q.put("name", new BasicDBObject('$in', ["中信泰富又一城一期","中信泰富又一城"])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictLocationGeo.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictLocationGeo.groovy index 512e54ee4..b25c63f1e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictLocationGeo.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictLocationGeo.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.gj.district +import com.fangjia.core.util.GeoUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils @@ -13,40 +14,36 @@ import org.bson.types.ObjectId class DistrictLocationGeo { - final DBCollection seaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "seaweed_gaode_pois", "192.168.0.235", 27018) - final DBCollection allSeaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "all_seaweed_gaode_pois", "192.168.0.235", 27018) + final DBCollection JD_SEAWEED_COLL = MongoUtils.getCollectionByName("business", "jingdong_community", "192.168.0.206",27020) + final DBCollection JD_GEO_COLL = MongoUtils.getCollectionByName("business", "jingdong_geo", "192.168.0.206",27020) static main(args) { def task = new DistrictLocationGeo() task.run()//通过经纬度,调用高德逆解析,并保存数据 -// task.run2();//通过地址,调用高德逆解析 +// def distance = GeoUtils.calculateDistance(130.573501, 45.752712, 130.573527,45.753287); +// println distance + } def run(){ - ExcelUtils.modify("D:\\数据\\京东\\失败.xlsx",{ d-> - if(d['E']){ - def jsonObject = GaoDeApiUtils.geocode(d['E']) - if(jsonObject){ - d["F"] = "高德" - def dbo = new BasicDBObject() - dbo.put("old_id", d['A']) - dbo.put("city", d['B']) - dbo.put("region", d['C']) - dbo.put("name", d['D']) - dbo.put("old_location", d['E']) - dbo.put("formatted_address", jsonObject.regeocode.formatted_address) - def addrComp = jsonObject.regeocode.addressComponent -// addrComp.remove("neighborhood") -// addrComp.remove("building") - dbo.put("addressComponent", addrComp) - dbo.put("pois", jsonObject.regeocode.pois) - dbo.put("roads", jsonObject.regeocode.roads) - dbo.put("roadinters", jsonObject.regeocode.roadinters) - dbo.put("aois", jsonObject.regeocode.aois) - dbo = cleanGeoData(dbo) - dbo = convertToDBObject(dbo) - println(dbo) - - allSeaweedGaodePoisColl.save(dbo) + ExcelUtils.multiThreadingModify("D:\\数据\\京东\\京东新增数据5.6万.xlsx",{ d-> + if(d['A']){ + def dbo = JD_SEAWEED_COLL.findOne(new BasicDBObject("_id", new ObjectId(d['A']))) + if(dbo && dbo.longitude){ + def jsonObject = GaoDeApiUtils.geocode(dbo.longitude+","+dbo.latitude) + if(jsonObject){ + d["F"] = "高德" +// def dbo = new BasicDBObject() + dbo.put("formatted_address", jsonObject.regeocode.formatted_address) + def addrComp = jsonObject.regeocode.addressComponent + dbo.put("addressComponent", addrComp) + dbo.put("pois", jsonObject.regeocode.pois) + dbo.put("roads", jsonObject.regeocode.roads) + dbo.put("roadinters", jsonObject.regeocode.roadinters) + dbo.put("aois", jsonObject.regeocode.aois) + dbo = cleanGeoData(dbo) + dbo = convertToDBObject(dbo) + JD_GEO_COLL.save(dbo) + } } } }) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/JindDongGeoTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/JindDongGeoTest.groovy new file mode 100644 index 000000000..2b0dfcb18 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/JindDongGeoTest.groovy @@ -0,0 +1,144 @@ +package com.fangjia.script.gj.district + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.BasicDBList +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import net.sf.json.JSONNull + +class JindDongGeoTest { + final DBCollection seaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "seaweed_gaode_pois", "192.168.0.235", 27018) + final DBCollection allSeaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "all_seaweed_gaode_pois", "192.168.0.235", 27018) + + static main(args) { + def task = new JindDongGeoTest() + task.run()//通过经纬度,调用高德逆解析,并保存数据 + } + + + def run(){ + ExcelUtils.modify("D:\\数据\\京东\\失败.xlsx",{ d-> + if(d['E']){ + def jsonObject = GaoDeApiUtils.geocode(d['E']) + if(jsonObject){ + d["F"] = "高德" + def dbo = new BasicDBObject() + dbo.put("old_id", d['A']) + dbo.put("city", d['B']) + dbo.put("region", d['C']) + dbo.put("name", d['D']) + dbo.put("old_location", d['E']) + dbo.put("formatted_address", jsonObject.regeocode.formatted_address) + def addrComp = jsonObject.regeocode.addressComponent +// addrComp.remove("neighborhood") +// addrComp.remove("building") + dbo.put("addressComponent", addrComp) + dbo.put("pois", jsonObject.regeocode.pois) + dbo.put("roads", jsonObject.regeocode.roads) + dbo.put("roadinters", jsonObject.regeocode.roadinters) + dbo.put("aois", jsonObject.regeocode.aois) + dbo = cleanGeoData(dbo) + dbo = convertToDBObject(dbo) + println(dbo) + + allSeaweedGaodePoisColl.save(dbo) + } + } + }) + + + } + + static BasicDBObject convertToDBObject(Map map) { + if (!map) return null + + BasicDBObject dbObject = new BasicDBObject() + map.each { key, value -> + def convertedValue = convertValue(value) + if (convertedValue != null) { + dbObject.put(key, convertedValue) + } + } + return dbObject + } + + static Object convertValue(Object value) { + if (value == null) return null + + switch (value) { + case Map: + return convertToDBObject(value as Map) + case List: + return convertList(value as List) + default: + return value + } + } + + static BasicDBList convertList(List list) { + BasicDBList dbList = new BasicDBList() + list.each { item -> + def convertedItem = convertValue(item) + if (convertedItem != null) { + dbList.add(convertedItem) + } + } + return dbList + } + + static def cleanGeoData(data) { + if (data == null) return null + + def result = [:] + + data.each { key, value -> + def cleanedValue = cleanValue(value) + if (cleanedValue != null && !isEmptyCollection(cleanedValue)) { + result[key] = cleanedValue + } + } + + return result + } + + static def cleanValue(value) { + if (value == null) return null + + switch (value) { + case Map: + def cleanedMap = [:] + value.each { k, v -> + def cleaned = cleanValue(v) + if (cleaned != null && !isEmptyCollection(cleaned)) { + cleanedMap[k] = cleaned + } + } + return cleanedMap ? cleanedMap : null + + case List: + def cleanedList = value.collect { cleanValue(it) } + .findAll { it != null && !isEmptyCollection(it) } + return cleanedList ? cleanedList : null + + case JSONNull: + return null + + default: + // 检查是否是包含JSONNull的字符串表示 + if (value instanceof String && value.contains("JSONNull")) { + return null + } + return value + } + } + + static def isEmptyCollection(obj) { + if (obj instanceof Map) return obj.isEmpty() + if (obj instanceof List) return obj.isEmpty() + return false + } + + +} From 91a2a291e5c762e5c82aaf882dc581de34680c4a Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Thu, 4 Dec 2025 20:00:42 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E6=9C=88=E5=BA=A6=E5=9D=87=E4=BB=B7?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fuzhou/DistrictSellStatBatch.groovy | 121 +++++++++++++++--- 1 file changed, 101 insertions(+), 20 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy index a4727d6fc..a06af6e14 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy @@ -4,11 +4,13 @@ import com.fangjia.core.util.DataUtil import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import com.mongodb.MongoClient import java.text.DecimalFormat import java.util.concurrent.CopyOnWriteArrayList @@ -26,6 +28,8 @@ class DistrictSellStatBatch extends BaseBatch { static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com") static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName("business", "fz_district", "mongo.db.fangjia.com") + public static final DBCollection COMMAND_COLL = MongoUtils.getCollectionByName("fangjia_log", "calc_command", "log.mongo.db.fangjia.com",52002) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) public static final DBCollection SEAWEED_MONGO_COLL = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com") @@ -37,14 +41,46 @@ class DistrictSellStatBatch extends BaseBatch { @Override void process(Object args) { - def months = DateUtils.getMonthList(202510) - months = months[0..(months.size()-1)] - println months - for(def month : months){ - def curDate = DateUtils.parseFromTightByPattern(month.toString()+"01", "yyyyMMdd") - calcMonthlyPrice(curDate) +// def months = DateUtils.getMonthList(202510) +// months = months[0..(months.size()-1)] +// println months +// for(def month : months){ +// def curDate = DateUtils.parseFromTightByPattern(month.toString()+"01", "yyyyMMdd") +// calcMonthlyPrice(curDate) +// } + + def date = new Date() + def day = date.getDate() + def hours = date.getHours() + if(day == 4 && hours<24){ + calcMonthlyPrice() //正常算一期 + }else{ + //根据calc_type算历史 + def q = new BasicDBObject("category" ,"sell_month") + def commands = COMMAND_COLL.find(q) + println "共 【"+commands.size()+"】 个小区重算 "+new Date() + for(def command : commands){ + def calc_type = command.calc_type + def id = command._id + if(calc_type==1){ + //找出第一次均价计算的时间 + def obj = CommonDatautils.queryFirstOrLastDistrictPrice(command.city, command.region, command.name, 1) + if(obj){ + def weekend = obj.weekend + def startMonth = DateUtils.formatDateByPattern(weekend, "yyyyMM").toInteger() + def months = DateUtils.getMonthList(startMonth) + months = months[1..(months.size()-1)] + println months + for(def month : months){ + def curDate = DateUtils.parseFromTightByPattern(month.toString()+"01", "yyyyMMdd") + calcMonthlyPriceByName(curDate, command.city, command.region, command.name) + } + //删除命令 + COMMAND_COLL.remove(new BasicDBObject("_id", id)) + } + } + } } -// calcMonthlyPrice() } @@ -69,7 +105,7 @@ class DistrictSellStatBatch extends BaseBatch { if(!curDate){ curDate = new Date() } - def citys = ["上海"]//CommonUtils.findAllCity() + def citys = CommonUtils.findAllCity() // citys.remove("上海") // citys.remove("福州") def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1).append("status", 1) @@ -92,15 +128,15 @@ class DistrictSellStatBatch extends BaseBatch { def districts = [] if(city == "福州"){ - if(day == 29){ +// if(day == 15){ districts = FZ_DIST_COLL.find(new BasicDBObject("city", city), keys).toList() - }else{ - districts = FZ_DIST_COLL.find(new BasicDBObject("city", city).append("m_date", new BasicDBObject('$gte',startMDate).append('$lt', endMDate)), keys).toList() - } +// }else{ +// districts = FZ_DIST_COLL.find(new BasicDBObject("city", city).append("m_date", new BasicDBObject('$gte',startMDate).append('$lt', endMDate)), keys).toList() +// } }else{ def q = new BasicDBObject("city", city).append("cat", "district").append("status", 0) // q.put("name", "越华路小区") -// if(day == 30){ +// if(day == 2){ districts = SEAWEED_MONGO_COLL.find(q, keys).batchSize(2000).toList() // }else{ // q.put("m_date", new BasicDBObject('$gte',startMDate).append('$lt', endMDate)) @@ -113,10 +149,10 @@ class DistrictSellStatBatch extends BaseBatch { def startDate = DateUtils.getFirstDayOfMonth(curDate, -1); def endDate = DateUtils.getLastDayOfMonth(curDate, -1); if(districts.size()>0){ - def weekendPriceMap = findPriceMap(city, startDate, endDate)//上个月小区均价 - def curMonthlyPriceMap = findMonthlyPrice(city, curMonth)//当月小区月均价 - def lastMonthlyPriceMap = findMonthlyPrice(city, monDate)//上个月小区月均价 - def lastYearPriceMap = findMonthlyPrice(city, yoyDate)//去年小区月均价 + def weekendPriceMap = findPriceMap(city, startDate, endDate, null, null)//上个月小区均价 + def curMonthlyPriceMap = findMonthlyPrice(city, curMonth, null, null)//当月小区月均价 + def lastMonthlyPriceMap = findMonthlyPrice(city, monDate, null, null)//上个月小区月均价 + def lastYearPriceMap = findMonthlyPrice(city, yoyDate, null, null)//去年小区月均价 if(weekendPriceMap){ running(curMonth, districts, weekendPriceMap, curMonthlyPriceMap, lastMonthlyPriceMap, lastYearPriceMap) } @@ -124,6 +160,44 @@ class DistrictSellStatBatch extends BaseBatch { } } + + def calcMonthlyPriceByName(curDate, city, region, name) { + //只有福州使用tob价格,改为所有城市都使用2B价格,除人工指导价除外 + if (!curDate) { + curDate = new Date() + } + + def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1).append("status", 1) + + def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(curDate, "yyyyMM"), -1, "yyyyMM")); + def monDate = getMomDate(curMonth) + def yoyDate = getYoyDate(curMonth) + + def q = new BasicDBObject("city", city).append("cat", "district").append("status", 0).append("region", region).append("name", name) + + def districts = SEAWEED_MONGO_COLL.find(q, keys).toList() + + + + def startDate = DateUtils.getFirstDayOfMonth(curDate, -1); + def endDate = DateUtils.getLastDayOfMonth(curDate, -1); + if (districts.size() > 0) { + def weekendPriceMap = findPriceMap(city, startDate, endDate, region, name)//上个月小区均价 + def curMonthlyPriceMap = findMonthlyPrice(city, curMonth, region, name)//当月小区月均价 + def lastMonthlyPriceMap = findMonthlyPrice(city, monDate, region, name)//上个月小区月均价 + def lastYearPriceMap = findMonthlyPrice(city, yoyDate, region, name)//去年小区月均价 + if (weekendPriceMap) { + running(curMonth, districts, weekendPriceMap, curMonthlyPriceMap, lastMonthlyPriceMap, lastYearPriceMap) + } + } + + } + + + + + + def running(districts){ def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(districts.size()) @@ -284,10 +358,13 @@ class DistrictSellStatBatch extends BaseBatch { } - def findPriceMap(city, startDate, endDate){ + def findPriceMap(city, startDate, endDate, region, name){ def q = new BasicDBObject() q.put("city", city) -// q.put("district_name","长春铺") + if(name){ + q.put("region", region) + q.put("district_name", name) + } q.put("weekend", new BasicDBObject('$lte', endDate).append('$gte', startDate)) def dbcursor = DISTRICT_STAT.find(q).sort(new BasicDBObject().append("weekend", -1)) def priceMap = [:] @@ -308,10 +385,14 @@ class DistrictSellStatBatch extends BaseBatch { return priceMap } - def findMonthlyPrice(city, monDate){ + def findMonthlyPrice(city, monDate, region, name){ def q = new BasicDBObject() q.put("city", city) q.put("date", monDate) + if(name){ + q.put("region", region) + q.put("district_name", name) + } def dbcursor = DISTRICT_SELL_STAT.find(q) def priceMap = [:] for(def d : dbcursor){