From 5033b0b4753fda8669e9e6c0795b85afce5fdff7 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Thu, 4 Dec 2025 11:07:29 +0800 Subject: [PATCH] update --- .../script/ccbevaluation/AddSzDealData.groovy | 2 +- .../ccbevaluation/CcbCommonUtils.groovy | 2 +- .../gj/district/DistrictLocationGeo.groovy | 53 +++---- .../script/gj/district/JindDongGeoTest.groovy | 144 ++++++++++++++++++ 4 files changed, 171 insertions(+), 30 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/JindDongGeoTest.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy index 9ebbbcaa2..0f76938ce 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy @@ -9,7 +9,7 @@ import com.mongodb.DBCollection * 深圳公众号成交数据录入 */ -def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳9月份成交数据.xlsx", 0, 1) +def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳10月份成交数据11.20.xlsx", 0, 1) def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 4dd815364..88194c547 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -443,7 +443,7 @@ class CcbCommonUtils { } // q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13 -// q.put("name","龙源星城") +// q.put("name","登高西路安置小区C区") // q.put("name_cat", 0); // q.put("name", new BasicDBObject('$in', ["中信泰富又一城一期","中信泰富又一城"])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictLocationGeo.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictLocationGeo.groovy index 512e54ee4..b25c63f1e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictLocationGeo.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictLocationGeo.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.gj.district +import com.fangjia.core.util.GeoUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils @@ -13,40 +14,36 @@ import org.bson.types.ObjectId class DistrictLocationGeo { - final DBCollection seaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "seaweed_gaode_pois", "192.168.0.235", 27018) - final DBCollection allSeaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "all_seaweed_gaode_pois", "192.168.0.235", 27018) + final DBCollection JD_SEAWEED_COLL = MongoUtils.getCollectionByName("business", "jingdong_community", "192.168.0.206",27020) + final DBCollection JD_GEO_COLL = MongoUtils.getCollectionByName("business", "jingdong_geo", "192.168.0.206",27020) static main(args) { def task = new DistrictLocationGeo() task.run()//通过经纬度,调用高德逆解析,并保存数据 -// task.run2();//通过地址,调用高德逆解析 +// def distance = GeoUtils.calculateDistance(130.573501, 45.752712, 130.573527,45.753287); +// println distance + } def run(){ - ExcelUtils.modify("D:\\数据\\京东\\失败.xlsx",{ d-> - if(d['E']){ - def jsonObject = GaoDeApiUtils.geocode(d['E']) - if(jsonObject){ - d["F"] = "高德" - def dbo = new BasicDBObject() - dbo.put("old_id", d['A']) - dbo.put("city", d['B']) - dbo.put("region", d['C']) - dbo.put("name", d['D']) - dbo.put("old_location", d['E']) - dbo.put("formatted_address", jsonObject.regeocode.formatted_address) - def addrComp = jsonObject.regeocode.addressComponent -// addrComp.remove("neighborhood") -// addrComp.remove("building") - dbo.put("addressComponent", addrComp) - dbo.put("pois", jsonObject.regeocode.pois) - dbo.put("roads", jsonObject.regeocode.roads) - dbo.put("roadinters", jsonObject.regeocode.roadinters) - dbo.put("aois", jsonObject.regeocode.aois) - dbo = cleanGeoData(dbo) - dbo = convertToDBObject(dbo) - println(dbo) - - allSeaweedGaodePoisColl.save(dbo) + ExcelUtils.multiThreadingModify("D:\\数据\\京东\\京东新增数据5.6万.xlsx",{ d-> + if(d['A']){ + def dbo = JD_SEAWEED_COLL.findOne(new BasicDBObject("_id", new ObjectId(d['A']))) + if(dbo && dbo.longitude){ + def jsonObject = GaoDeApiUtils.geocode(dbo.longitude+","+dbo.latitude) + if(jsonObject){ + d["F"] = "高德" +// def dbo = new BasicDBObject() + dbo.put("formatted_address", jsonObject.regeocode.formatted_address) + def addrComp = jsonObject.regeocode.addressComponent + dbo.put("addressComponent", addrComp) + dbo.put("pois", jsonObject.regeocode.pois) + dbo.put("roads", jsonObject.regeocode.roads) + dbo.put("roadinters", jsonObject.regeocode.roadinters) + dbo.put("aois", jsonObject.regeocode.aois) + dbo = cleanGeoData(dbo) + dbo = convertToDBObject(dbo) + JD_GEO_COLL.save(dbo) + } } } }) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/JindDongGeoTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/JindDongGeoTest.groovy new file mode 100644 index 000000000..2b0dfcb18 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/JindDongGeoTest.groovy @@ -0,0 +1,144 @@ +package com.fangjia.script.gj.district + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.BasicDBList +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import net.sf.json.JSONNull + +class JindDongGeoTest { + final DBCollection seaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "seaweed_gaode_pois", "192.168.0.235", 27018) + final DBCollection allSeaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "all_seaweed_gaode_pois", "192.168.0.235", 27018) + + static main(args) { + def task = new JindDongGeoTest() + task.run()//通过经纬度,调用高德逆解析,并保存数据 + } + + + def run(){ + ExcelUtils.modify("D:\\数据\\京东\\失败.xlsx",{ d-> + if(d['E']){ + def jsonObject = GaoDeApiUtils.geocode(d['E']) + if(jsonObject){ + d["F"] = "高德" + def dbo = new BasicDBObject() + dbo.put("old_id", d['A']) + dbo.put("city", d['B']) + dbo.put("region", d['C']) + dbo.put("name", d['D']) + dbo.put("old_location", d['E']) + dbo.put("formatted_address", jsonObject.regeocode.formatted_address) + def addrComp = jsonObject.regeocode.addressComponent +// addrComp.remove("neighborhood") +// addrComp.remove("building") + dbo.put("addressComponent", addrComp) + dbo.put("pois", jsonObject.regeocode.pois) + dbo.put("roads", jsonObject.regeocode.roads) + dbo.put("roadinters", jsonObject.regeocode.roadinters) + dbo.put("aois", jsonObject.regeocode.aois) + dbo = cleanGeoData(dbo) + dbo = convertToDBObject(dbo) + println(dbo) + + allSeaweedGaodePoisColl.save(dbo) + } + } + }) + + + } + + static BasicDBObject convertToDBObject(Map map) { + if (!map) return null + + BasicDBObject dbObject = new BasicDBObject() + map.each { key, value -> + def convertedValue = convertValue(value) + if (convertedValue != null) { + dbObject.put(key, convertedValue) + } + } + return dbObject + } + + static Object convertValue(Object value) { + if (value == null) return null + + switch (value) { + case Map: + return convertToDBObject(value as Map) + case List: + return convertList(value as List) + default: + return value + } + } + + static BasicDBList convertList(List list) { + BasicDBList dbList = new BasicDBList() + list.each { item -> + def convertedItem = convertValue(item) + if (convertedItem != null) { + dbList.add(convertedItem) + } + } + return dbList + } + + static def cleanGeoData(data) { + if (data == null) return null + + def result = [:] + + data.each { key, value -> + def cleanedValue = cleanValue(value) + if (cleanedValue != null && !isEmptyCollection(cleanedValue)) { + result[key] = cleanedValue + } + } + + return result + } + + static def cleanValue(value) { + if (value == null) return null + + switch (value) { + case Map: + def cleanedMap = [:] + value.each { k, v -> + def cleaned = cleanValue(v) + if (cleaned != null && !isEmptyCollection(cleaned)) { + cleanedMap[k] = cleaned + } + } + return cleanedMap ? cleanedMap : null + + case List: + def cleanedList = value.collect { cleanValue(it) } + .findAll { it != null && !isEmptyCollection(it) } + return cleanedList ? cleanedList : null + + case JSONNull: + return null + + default: + // 检查是否是包含JSONNull的字符串表示 + if (value instanceof String && value.contains("JSONNull")) { + return null + } + return value + } + } + + static def isEmptyCollection(obj) { + if (obj instanceof Map) return obj.isEmpty() + if (obj instanceof List) return obj.isEmpty() + return false + } + + +}