This commit is contained in:
2025-11-19 11:45:27 +08:00
parent 1a15f8d2cf
commit 3ffa7fb957
10 changed files with 224 additions and 42 deletions
@@ -813,8 +813,9 @@ class CommonDatautils {
// q.put("status",new BasicDBObject('$ne', 0)) // q.put("status",new BasicDBObject('$ne', 0))
q.put("cat", cat); q.put("cat", cat);
q.put("city", city) q.put("city", city)
// q.put("loop_line_info",new BasicDBObject('$exists',false)) // q.put("loop_line_info",new BasicDBObject('$exists',false))
// q.put("name","经纬天地园二期") // q.put("name","扬州北街8号")
DBObject keys = new BasicDBObject(); DBObject keys = new BasicDBObject();
if(fields){ if(fields){
@@ -211,6 +211,7 @@ class DistrictInfoExportBatch extends BaseBatch{
def getFzDistricts(def city, def mdate, def endMDate){ def getFzDistricts(def city, def mdate, def endMDate){
def q = new BasicDBObject() def q = new BasicDBObject()
q.put("city", city) q.put("city", city)
if(mdate){ if(mdate){
q.put("m_date", new BasicDBObject('$gte', mdate).append('$lt', endMDate)) q.put("m_date", new BasicDBObject('$gte', mdate).append('$lt', endMDate))
} }
@@ -99,6 +99,7 @@ class DistrictPriceExportBatch extends BaseBatch{
def getFzDistricts(def city, def mdate, def endMDate){ def getFzDistricts(def city, def mdate, def endMDate){
def q = new BasicDBObject() def q = new BasicDBObject()
q.put("city", city) q.put("city", city)
if(mdate){ if(mdate){
q.put("m_date", new BasicDBObject('$gte', mdate).append('$lt', endMDate)) q.put("m_date", new BasicDBObject('$gte', mdate).append('$lt', endMDate))
} }
@@ -831,7 +831,7 @@ class OpenApiUtils {
static def addrVerifyGuangda(city, address){ static def addrVerifyGuangda(city, address){
address = URLEncoder.encode(address, "utf-8"); address = URLEncoder.encode(address, "utf-8");
def url = "http://192.168.0.100:32080/FangJiaAddressService/cebEnterpriseVillageIdentify?address=${address}" def url = "http://192.168.0.100:32080/FangJiaAddressService/cebEnterpriseVillageIdentify?address=${address}"
// def url = "http://localhost:8080/fangjia-open/addrSvc/njcbVillageIdentify?address=${address}&type=${type}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" // def url = "http://localhost:8080/FangJiaAddressService/cebEnterpriseVillageIdentify?address=${address}"
if(city){ if(city){
url = url + "&city=${city}" url = url + "&city=${city}"
} }
@@ -847,23 +847,23 @@ class OpenApiUtils {
} }
static def addrVerifyShBank(city, address){ // static def addrVerifyShBank(city, address){
address = URLEncoder.encode(address, "utf-8"); // address = URLEncoder.encode(address, "utf-8");
// def url = "http://192.168.0.100:32080/FangJiaAddressService/cebEnterpriseVillageIdentify?address=${address}" //// def url = "http://192.168.0.100:32080/FangJiaAddressService/cebEnterpriseVillageIdentify?address=${address}"
def url = "http://localhost:8080/addrSvc/boscEnterpriseVillageIdentify?address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" // def url = "http://localhost:8080/addrSvc/boscEnterpriseVillageIdentify?address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
if(city){ // if(city){
url = url + "&city=${city}" // url = url + "&city=${city}"
} // }
String content = new HttpRequest().get(url, "utf-8"); // String content = new HttpRequest().get(url, "utf-8");
if(content){ // if(content){
def doc = new JsonSlurper().parseText(content) // def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==200 && doc.result){ // if(doc && doc.code ==200 && doc.result){
def dtos = doc.result // def dtos = doc.result
return dtos // return dtos
} // }
} // }
return null; // return null;
} // }
static def addrVerifyBase(city, address){ static def addrVerifyBase(city, address){
@@ -33,8 +33,8 @@ class DevUpdateTestBatch {
static main(args) { static main(args) {
def task = new DevUpdateTestBatch() def task = new DevUpdateTestBatch()
def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx
task.addDev(path) // task.addDev(path)
// task.addDevAddress(path) task.addDevAddress(path)
// //
// task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是") // task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是")
@@ -3,32 +3,157 @@ package com.fangjia.script.gj.district
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBList
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import net.sf.json.JSONNull
import org.bson.types.ObjectId import org.bson.types.ObjectId
class DistrictLocationGeo { class DistrictLocationGeo {
final DBCollection seaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "seaweed_gaode_pois", "192.168.0.235", 27018) final DBCollection seaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "seaweed_gaode_pois", "192.168.0.235", 27018)
final DBCollection allSeaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "all_seaweed_gaode_pois", "192.168.0.235", 27018)
static main(args) { static main(args) {
def task = new DistrictLocationGeo() def task = new DistrictLocationGeo()
task.run()//调用高德逆解析,并保存数据 task.run()//通过经纬度,调用高德逆解析,并保存数据
// task.run2();//通过地址,调用高德逆解析
} }
def run(){ def run(){
ExcelUtils.modify("D:\\数据\\京东\\需逆解析数据-test.xlsx",{ d-> ExcelUtils.modify("D:\\数据\\京东\\失败.xlsx",{ d->
def jsonObject = GaoDeApiUtils.geocode(d['C']) if(d['E']){
def jsonObject = GaoDeApiUtils.geocode(d['E'])
if(jsonObject){
d["F"] = "高德"
def dbo = new BasicDBObject()
dbo.put("old_id", d['A'])
dbo.put("city", d['B'])
dbo.put("region", d['C'])
dbo.put("name", d['D'])
dbo.put("old_location", d['E'])
dbo.put("formatted_address", jsonObject.regeocode.formatted_address)
def addrComp = jsonObject.regeocode.addressComponent
// addrComp.remove("neighborhood")
// addrComp.remove("building")
dbo.put("addressComponent", addrComp)
dbo.put("pois", jsonObject.regeocode.pois)
dbo.put("roads", jsonObject.regeocode.roads)
dbo.put("roadinters", jsonObject.regeocode.roadinters)
dbo.put("aois", jsonObject.regeocode.aois)
dbo = cleanGeoData(dbo)
dbo = convertToDBObject(dbo)
println(dbo)
allSeaweedGaodePoisColl.save(dbo)
}
}
})
}
static BasicDBObject convertToDBObject(Map map) {
if (!map) return null
BasicDBObject dbObject = new BasicDBObject()
map.each { key, value ->
def convertedValue = convertValue(value)
if (convertedValue != null) {
dbObject.put(key, convertedValue)
}
}
return dbObject
}
static Object convertValue(Object value) {
if (value == null) return null
switch (value) {
case Map:
return convertToDBObject(value as Map)
case List:
return convertList(value as List)
default:
return value
}
}
static BasicDBList convertList(List list) {
BasicDBList dbList = new BasicDBList()
list.each { item ->
def convertedItem = convertValue(item)
if (convertedItem != null) {
dbList.add(convertedItem)
}
}
return dbList
}
static def cleanGeoData(data) {
if (data == null) return null
def result = [:]
data.each { key, value ->
def cleanedValue = cleanValue(value)
if (cleanedValue != null && !isEmptyCollection(cleanedValue)) {
result[key] = cleanedValue
}
}
return result
}
static def cleanValue(value) {
if (value == null) return null
switch (value) {
case Map:
def cleanedMap = [:]
value.each { k, v ->
def cleaned = cleanValue(v)
if (cleaned != null && !isEmptyCollection(cleaned)) {
cleanedMap[k] = cleaned
}
}
return cleanedMap ? cleanedMap : null
case List:
def cleanedList = value.collect { cleanValue(it) }
.findAll { it != null && !isEmptyCollection(it) }
return cleanedList ? cleanedList : null
case JSONNull:
return null
default:
// 检查是否是包含JSONNull的字符串表示
if (value instanceof String && value.contains("JSONNull")) {
return null
}
return value
}
}
static def isEmptyCollection(obj) {
if (obj instanceof Map) return obj.isEmpty()
if (obj instanceof List) return obj.isEmpty()
return false
}
def run2(){
ExcelUtils.modify("D:\\数据\\昆仑\\263未出居委数据.xlsx",{ d->
def jsonObject = GaoDeApiUtils.geocodeByAddress(d['C'], d['B'])
Thread.sleep(1000)
if(jsonObject){ if(jsonObject){
def dbo = new BasicDBObject() def rets = jsonObject.geocodes
dbo.put("old_id", d['B']) if(rets){
dbo.put("old_location", d['C']) def dbo = rets[0]
dbo.put("formatted_address", jsonObject.regeocode.formatted_address) if(dbo){
dbo.put("addressComponent", jsonObject.regeocode.addressComponent) d['D'] = dbo.formatted_address
dbo.put("pois", jsonObject.regeocode.pois) d['E'] = dbo.level
dbo.put("roads", jsonObject.regeocode.roads) }
dbo.put("roadinters", jsonObject.regeocode.roadinters) }
dbo.put("aois", jsonObject.regeocode.aois)
seaweedGaodePoisColl.save(dbo)
} }
}) })
} }
@@ -355,7 +355,7 @@ class FangjiaClientUtils {
address = address.toString().replace("{","") address = address.toString().replace("{","")
int retryCount = 0 int retryCount = 0
while (retryCount < 3) { while (retryCount < 3) {
url = "http://192.168.0.93:8080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true&fields=dev_confi" url = "http://192.168.0.100:32080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true&fields=dev_confi"
if(city){ if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8") url = url +"&city="+URLEncoder.encode(city,"UTF-8")
} }
@@ -1,19 +1,36 @@
package com.fangjia.script.gj.village package com.fangjia.script.gj.village
import com.fangjia.script.gj.OpenApiUtils import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
class KunlunVillage { class KunlunVillage {
static main(args) { static main(args) {
def task = new KunlunVillage(); def task = new KunlunVillage();
task.run() task.run()
// task.addrCheck()
} }
def run(){ def addrCheck(){
ExcelUtils.multiThreadingModify("D:\\数据\\昆仑\\3987户需批量更新涉农信息明细-20251023-结果.xlsx",{ d->
if(!d['R'] && d['B']){ ExcelUtils.modify("D:\\数据\\昆仑\\2025年11月18日批次跑批涉农地址-发估家.xlsx",{ d->
def dto = OpenApiUtils.addrVerifyGuangda(null, d['B']) def obj = FangjiaClientUtils.formatNinelevel(null, d["C"])
if(obj){
println obj
d['AB'] = obj.addressAssessment.msg
}
})
}
def run(){
def index = 0
ExcelUtils.modify("D:\\数据\\昆仑\\2025年11月18日批次跑批涉农地址-发估家.xlsx",{ d->
index++
if(!d['R'] && d['C']){
println index
def dto = OpenApiUtils.addrVerifyBase(null, d['C'])
if(dto){ if(dto){
// def dto = jsonObj.result // def dto = jsonObj.result
d['F'] = dto.province d['F'] = dto.province
@@ -414,7 +414,7 @@ public class ExcelUtils{
def data = ExcelUtils.read(fileName, srcSheetIndex, 0) def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
def body = data.subList(1, data.size()) def body = data.subList(1, data.size())
def nThread = Runtime.getRuntime().availableProcessors() * 6 def nThread = Runtime.getRuntime().availableProcessors() * 8
println nThread println nThread
ExecutorService executorService = Executors.newFixedThreadPool(nThread) ExecutorService executorService = Executors.newFixedThreadPool(nThread)
final AtomicInteger counter = new AtomicInteger(0) final AtomicInteger counter = new AtomicInteger(0)
@@ -13,7 +13,7 @@ import java.util.zip.GZIPInputStream
class GaoDeApiUtils { class GaoDeApiUtils {
//94c74c60e102972a8d17b7b75d7403be //94c74c60e102972a8d17b7b75d7403be
def static GAO_DE_AK = "a34894d151fffff94f9a33bae3d5565c" //宝剑:c6fcd4bfbc40d73ee762b8115dca3390 def static GAO_DE_AK = "947cd51c2f3110a93cbd755f697a526d" //宝剑:c6fcd4bfbc40d73ee762b8115dca3390
//15021384995:dfc2245bac8a7b2d5b62e060e59d4eca //15021384995:dfc2245bac8a7b2d5b62e060e59d4eca
/** /**
@@ -30,7 +30,10 @@ class GaoDeApiUtils {
params.put("location", location) params.put("location", location)
params.put("extensions","all") params.put("extensions","all")
params.put("radius", "3000") params.put("radius", "3000")
params.put("poitype", "120000|120200|120203|120300|120301|120302|120303") // params.put("poitype", "120000|120200|120203|120300|120301|120302|120303")//纯住宅类型
//京东测试允许类型
params.put("poitype", "100000|100100|100101|100102|100103|100104|100105|100200|100201|120000|120200|120203|120300|120301|120302|120303|120304|190000|190100|190400|190401|190402|190403|190500|190600|190700")
String content String content
def stop = true def stop = true
JSONObject jsonObj = null JSONObject jsonObj = null
@@ -57,6 +60,40 @@ class GaoDeApiUtils {
return jsonObj return jsonObj
} }
def static geocodeByAddress(String city, String address) {
Map<String, String> params = new HashMap<String, String>()
params.put("key", GAO_DE_AK)
params.put("address", address)
params.put("batch", "true")
params.put("city", city)
// params.put("poitype", "120000|120200|120203|120300|120301|120302|120303")
String content
def stop = true
JSONObject jsonObj = null
def errorCount = 1
while (stop){
try {
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/geo", params, "utf-8")
} catch (Exception e) {
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/geo", params, "utf-8")
}
jsonObj = JSONObject.fromObject(content)
println jsonObj
if(jsonObj.status!="1"){
println jsonObj
errorCount ++
Thread.sleep(3000)
if(errorCount>3){
stop = false
}
}else{
stop = false
}
}
return jsonObj
}
def static geo(city, address) { def static geo(city, address) {
Map<String, String> params = new HashMap<String, String>() Map<String, String> params = new HashMap<String, String>()
params.put("key", "a34894d151fffff94f9a33bae3d5565c")//许镕杰 params.put("key", "a34894d151fffff94f9a33bae3d5565c")//许镕杰