This commit is contained in:
2025-12-04 11:07:29 +08:00
parent 88cd67a3fe
commit 5033b0b475
4 changed files with 171 additions and 30 deletions
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳9月份成交数据.xlsx", 0, 1)
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳10月份成交数据11.20.xlsx", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -443,7 +443,7 @@ class CcbCommonUtils {
}
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13
// q.put("name","龙源星城")
// q.put("name","登高西路安置小区C区")
// q.put("name_cat", 0);
// q.put("name", new BasicDBObject('$in', ["中信泰富又一城一期","中信泰富又一城"]))
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.district
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
@@ -13,40 +14,36 @@ import org.bson.types.ObjectId
class DistrictLocationGeo {
final DBCollection seaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "seaweed_gaode_pois", "192.168.0.235", 27018)
final DBCollection allSeaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "all_seaweed_gaode_pois", "192.168.0.235", 27018)
final DBCollection JD_SEAWEED_COLL = MongoUtils.getCollectionByName("business", "jingdong_community", "192.168.0.206",27020)
final DBCollection JD_GEO_COLL = MongoUtils.getCollectionByName("business", "jingdong_geo", "192.168.0.206",27020)
static main(args) {
def task = new DistrictLocationGeo()
task.run()//通过经纬度,调用高德逆解析,并保存数据
// task.run2();//通过地址,调用高德逆解析
// def distance = GeoUtils.calculateDistance(130.573501, 45.752712, 130.573527,45.753287);
// println distance
}
def run(){
ExcelUtils.modify("D:\\数据\\京东\\失败.xlsx",{ d->
if(d['E']){
def jsonObject = GaoDeApiUtils.geocode(d['E'])
if(jsonObject){
d["F"] = "高德"
def dbo = new BasicDBObject()
dbo.put("old_id", d['A'])
dbo.put("city", d['B'])
dbo.put("region", d['C'])
dbo.put("name", d['D'])
dbo.put("old_location", d['E'])
dbo.put("formatted_address", jsonObject.regeocode.formatted_address)
def addrComp = jsonObject.regeocode.addressComponent
// addrComp.remove("neighborhood")
// addrComp.remove("building")
dbo.put("addressComponent", addrComp)
dbo.put("pois", jsonObject.regeocode.pois)
dbo.put("roads", jsonObject.regeocode.roads)
dbo.put("roadinters", jsonObject.regeocode.roadinters)
dbo.put("aois", jsonObject.regeocode.aois)
dbo = cleanGeoData(dbo)
dbo = convertToDBObject(dbo)
println(dbo)
allSeaweedGaodePoisColl.save(dbo)
ExcelUtils.multiThreadingModify("D:\\数据\\京东\\京东新增数据5.6万.xlsx",{ d->
if(d['A']){
def dbo = JD_SEAWEED_COLL.findOne(new BasicDBObject("_id", new ObjectId(d['A'])))
if(dbo && dbo.longitude){
def jsonObject = GaoDeApiUtils.geocode(dbo.longitude+","+dbo.latitude)
if(jsonObject){
d["F"] = "高德"
// def dbo = new BasicDBObject()
dbo.put("formatted_address", jsonObject.regeocode.formatted_address)
def addrComp = jsonObject.regeocode.addressComponent
dbo.put("addressComponent", addrComp)
dbo.put("pois", jsonObject.regeocode.pois)
dbo.put("roads", jsonObject.regeocode.roads)
dbo.put("roadinters", jsonObject.regeocode.roadinters)
dbo.put("aois", jsonObject.regeocode.aois)
dbo = cleanGeoData(dbo)
dbo = convertToDBObject(dbo)
JD_GEO_COLL.save(dbo)
}
}
}
})
@@ -0,0 +1,144 @@
package com.fangjia.script.gj.district
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBList
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONNull
class JindDongGeoTest {
final DBCollection seaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "seaweed_gaode_pois", "192.168.0.235", 27018)
final DBCollection allSeaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "all_seaweed_gaode_pois", "192.168.0.235", 27018)
static main(args) {
def task = new JindDongGeoTest()
task.run()//通过经纬度,调用高德逆解析,并保存数据
}
def run(){
ExcelUtils.modify("D:\\数据\\京东\\失败.xlsx",{ d->
if(d['E']){
def jsonObject = GaoDeApiUtils.geocode(d['E'])
if(jsonObject){
d["F"] = "高德"
def dbo = new BasicDBObject()
dbo.put("old_id", d['A'])
dbo.put("city", d['B'])
dbo.put("region", d['C'])
dbo.put("name", d['D'])
dbo.put("old_location", d['E'])
dbo.put("formatted_address", jsonObject.regeocode.formatted_address)
def addrComp = jsonObject.regeocode.addressComponent
// addrComp.remove("neighborhood")
// addrComp.remove("building")
dbo.put("addressComponent", addrComp)
dbo.put("pois", jsonObject.regeocode.pois)
dbo.put("roads", jsonObject.regeocode.roads)
dbo.put("roadinters", jsonObject.regeocode.roadinters)
dbo.put("aois", jsonObject.regeocode.aois)
dbo = cleanGeoData(dbo)
dbo = convertToDBObject(dbo)
println(dbo)
allSeaweedGaodePoisColl.save(dbo)
}
}
})
}
static BasicDBObject convertToDBObject(Map map) {
if (!map) return null
BasicDBObject dbObject = new BasicDBObject()
map.each { key, value ->
def convertedValue = convertValue(value)
if (convertedValue != null) {
dbObject.put(key, convertedValue)
}
}
return dbObject
}
static Object convertValue(Object value) {
if (value == null) return null
switch (value) {
case Map:
return convertToDBObject(value as Map)
case List:
return convertList(value as List)
default:
return value
}
}
static BasicDBList convertList(List list) {
BasicDBList dbList = new BasicDBList()
list.each { item ->
def convertedItem = convertValue(item)
if (convertedItem != null) {
dbList.add(convertedItem)
}
}
return dbList
}
static def cleanGeoData(data) {
if (data == null) return null
def result = [:]
data.each { key, value ->
def cleanedValue = cleanValue(value)
if (cleanedValue != null && !isEmptyCollection(cleanedValue)) {
result[key] = cleanedValue
}
}
return result
}
static def cleanValue(value) {
if (value == null) return null
switch (value) {
case Map:
def cleanedMap = [:]
value.each { k, v ->
def cleaned = cleanValue(v)
if (cleaned != null && !isEmptyCollection(cleaned)) {
cleanedMap[k] = cleaned
}
}
return cleanedMap ? cleanedMap : null
case List:
def cleanedList = value.collect { cleanValue(it) }
.findAll { it != null && !isEmptyCollection(it) }
return cleanedList ? cleanedList : null
case JSONNull:
return null
default:
// 检查是否是包含JSONNull的字符串表示
if (value instanceof String && value.contains("JSONNull")) {
return null
}
return value
}
}
static def isEmptyCollection(obj) {
if (obj instanceof Map) return obj.isEmpty()
if (obj instanceof List) return obj.isEmpty()
return false
}
}