diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index c64ee5c4b..736ff631d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -813,8 +813,9 @@ class CommonDatautils { // q.put("status",new BasicDBObject('$ne', 0)) q.put("cat", cat); q.put("city", city) + // q.put("loop_line_info",new BasicDBObject('$exists',false)) -// q.put("name","经纬天地园二期") +// q.put("name","扬州北街8号") DBObject keys = new BasicDBObject(); if(fields){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy index d49b696f3..9c7fa617d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy @@ -211,6 +211,7 @@ class DistrictInfoExportBatch extends BaseBatch{ def getFzDistricts(def city, def mdate, def endMDate){ def q = new BasicDBObject() q.put("city", city) + if(mdate){ q.put("m_date", new BasicDBObject('$gte', mdate).append('$lt', endMDate)) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy index 271b157aa..862fd5f8f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy @@ -99,6 +99,7 @@ class DistrictPriceExportBatch extends BaseBatch{ def getFzDistricts(def city, def mdate, def endMDate){ def q = new BasicDBObject() q.put("city", city) + if(mdate){ q.put("m_date", new BasicDBObject('$gte', mdate).append('$lt', endMDate)) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy index 4da8b5658..a4727d6fc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy @@ -37,7 +37,7 @@ class DistrictSellStatBatch extends BaseBatch { @Override void process(Object args) { - def months = DateUtils.getMonthList(202209) + def months = DateUtils.getMonthList(202510) months = months[0..(months.size()-1)] println months for(def month : months){ @@ -69,9 +69,9 @@ class DistrictSellStatBatch extends BaseBatch { if(!curDate){ curDate = new Date() } - def citys = CommonUtils.findAllCity() - citys.remove("上海") - citys.remove("福州") + def citys = ["上海"]//CommonUtils.findAllCity() +// citys.remove("上海") +// citys.remove("福州") def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1).append("status", 1) def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(curDate)) @@ -89,9 +89,7 @@ class DistrictSellStatBatch extends BaseBatch { for(def city : citys){ index++ - if(index < 242){ - continue - } + def districts = [] if(city == "福州"){ if(day == 29){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy index d2c1f91e1..7dfbdfcb1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy @@ -831,7 +831,7 @@ class OpenApiUtils { static def addrVerifyGuangda(city, address){ address = URLEncoder.encode(address, "utf-8"); def url = "http://192.168.0.100:32080/FangJiaAddressService/cebEnterpriseVillageIdentify?address=${address}" -// def url = "http://localhost:8080/fangjia-open/addrSvc/njcbVillageIdentify?address=${address}&type=${type}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" +// def url = "http://localhost:8080/FangJiaAddressService/cebEnterpriseVillageIdentify?address=${address}" if(city){ url = url + "&city=${city}" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy index 45332f6f1..6650ea864 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy @@ -33,8 +33,8 @@ class DevUpdateTestBatch { static main(args) { def task = new DevUpdateTestBatch() def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx - task.addDev(path) -// task.addDevAddress(path) +// task.addDev(path) + task.addDevAddress(path) // // task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictLocationGeo.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictLocationGeo.groovy index eefe4d587..512e54ee4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictLocationGeo.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictLocationGeo.groovy @@ -3,32 +3,160 @@ package com.fangjia.script.gj.district import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.BasicDBList import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import net.sf.json.JSONNull +import org.bson.types.ObjectId + + class DistrictLocationGeo { final DBCollection seaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "seaweed_gaode_pois", "192.168.0.235", 27018) + final DBCollection allSeaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "all_seaweed_gaode_pois", "192.168.0.235", 27018) static main(args) { def task = new DistrictLocationGeo() - task.run()//调用高德逆解析,并保存数据 + task.run()//通过经纬度,调用高德逆解析,并保存数据 +// task.run2();//通过地址,调用高德逆解析 } def run(){ - ExcelUtils.multiThreadingModify("D:\\数据\\京东\\需逆解析数据.xlsx", { d -> - def jsonObject = GaoDeApiUtils.geocode(d['C']) + ExcelUtils.modify("D:\\数据\\京东\\失败.xlsx",{ d-> + if(d['E']){ + def jsonObject = GaoDeApiUtils.geocode(d['E']) + if(jsonObject){ + d["F"] = "高德" + def dbo = new BasicDBObject() + dbo.put("old_id", d['A']) + dbo.put("city", d['B']) + dbo.put("region", d['C']) + dbo.put("name", d['D']) + dbo.put("old_location", d['E']) + dbo.put("formatted_address", jsonObject.regeocode.formatted_address) + def addrComp = jsonObject.regeocode.addressComponent +// addrComp.remove("neighborhood") +// addrComp.remove("building") + dbo.put("addressComponent", addrComp) + dbo.put("pois", jsonObject.regeocode.pois) + dbo.put("roads", jsonObject.regeocode.roads) + dbo.put("roadinters", jsonObject.regeocode.roadinters) + dbo.put("aois", jsonObject.regeocode.aois) + dbo = cleanGeoData(dbo) + dbo = convertToDBObject(dbo) + println(dbo) + + allSeaweedGaodePoisColl.save(dbo) + } + } + }) + + + } + + static BasicDBObject convertToDBObject(Map map) { + if (!map) return null + + BasicDBObject dbObject = new BasicDBObject() + map.each { key, value -> + def convertedValue = convertValue(value) + if (convertedValue != null) { + dbObject.put(key, convertedValue) + } + } + return dbObject + } + + static Object convertValue(Object value) { + if (value == null) return null + + switch (value) { + case Map: + return convertToDBObject(value as Map) + case List: + return convertList(value as List) + default: + return value + } + } + + static BasicDBList convertList(List list) { + BasicDBList dbList = new BasicDBList() + list.each { item -> + def convertedItem = convertValue(item) + if (convertedItem != null) { + dbList.add(convertedItem) + } + } + return dbList + } + + static def cleanGeoData(data) { + if (data == null) return null + + def result = [:] + + data.each { key, value -> + def cleanedValue = cleanValue(value) + if (cleanedValue != null && !isEmptyCollection(cleanedValue)) { + result[key] = cleanedValue + } + } + + return result + } + + static def cleanValue(value) { + if (value == null) return null + + switch (value) { + case Map: + def cleanedMap = [:] + value.each { k, v -> + def cleaned = cleanValue(v) + if (cleaned != null && !isEmptyCollection(cleaned)) { + cleanedMap[k] = cleaned + } + } + return cleanedMap ? cleanedMap : null + + case List: + def cleanedList = value.collect { cleanValue(it) } + .findAll { it != null && !isEmptyCollection(it) } + return cleanedList ? cleanedList : null + + case JSONNull: + return null + + default: + // 检查是否是包含JSONNull的字符串表示 + if (value instanceof String && value.contains("JSONNull")) { + return null + } + return value + } + } + + static def isEmptyCollection(obj) { + if (obj instanceof Map) return obj.isEmpty() + if (obj instanceof List) return obj.isEmpty() + return false + } + + def run2(){ + ExcelUtils.modify("D:\\数据\\昆仑\\263未出居委数据.xlsx",{ d-> + def jsonObject = GaoDeApiUtils.geocodeByAddress(d['C'], d['B']) + Thread.sleep(1000) def count = 0 if (jsonObject) { - def dbo = new BasicDBObject() - dbo.put("old_id", d['B']) - dbo.put("old_location", d['C']) - dbo.put("formatted_address", jsonObject.regeocode.formatted_address) - dbo.put("addressComponent", jsonObject.regeocode.addressComponent) - dbo.put("pois", jsonObject.regeocode.pois) - dbo.put("roads", jsonObject.regeocode.roads) - dbo.put("roadinters", jsonObject.regeocode.roadinters) - dbo.put("aois", jsonObject.regeocode.aois) - seaweedGaodePoisColl.save(dbo) + def rets = jsonObject.geocodes + if(rets){ + def dbo = rets[0] + if(dbo){ + d['D'] = dbo.formatted_address + d['E'] = dbo.level + } + } count = jsonObject.regeocode.pois.size() } d['D'] = count diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index ad2d5309a..0dfe2daab 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -355,7 +355,7 @@ class FangjiaClientUtils { address = address.toString().replace("{","") int retryCount = 0 while (retryCount < 3) { - url = "http://192.168.0.93:8080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true&fields=dev_confi" + url = "http://192.168.0.100:32080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true&fields=dev_confi" if(city){ url = url +"&city="+URLEncoder.encode(city,"UTF-8") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/KunlunVillage.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/KunlunVillage.groovy index d8b140529..86bb2ea86 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/KunlunVillage.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/KunlunVillage.groovy @@ -1,19 +1,36 @@ package com.fangjia.script.gj.village import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.util.ExcelUtils class KunlunVillage { static main(args) { def task = new KunlunVillage(); task.run() +// task.addrCheck() } - def run(){ - ExcelUtils.multiThreadingModify("D:\\数据\\昆仑\\3987户需批量更新涉农信息明细-20251023-结果.xlsx",{ d-> + def addrCheck(){ - if(!d['R'] && d['B']){ - def dto = OpenApiUtils.addrVerifyGuangda(null, d['B']) + ExcelUtils.modify("D:\\数据\\昆仑\\2025年11月18日批次跑批涉农地址-发估家.xlsx",{ d-> + def obj = FangjiaClientUtils.formatNinelevel(null, d["C"]) + if(obj){ + println obj + d['AB'] = obj.addressAssessment.msg + } + }) + + } + + + def run(){ + def index = 0 + ExcelUtils.modify("D:\\数据\\昆仑\\2025年11月18日批次跑批涉农地址-发估家.xlsx",{ d-> + index++ + if(!d['R'] && d['C']){ + println index + def dto = OpenApiUtils.addrVerifyBase(null, d['C']) if(dto){ // def dto = jsonObj.result d['F'] = dto.province diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy index 4e23f59e1..5a037dba9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy @@ -56,7 +56,7 @@ class CZDataBatch extends BaseBatch{ def weekend = DateUtils.getFirstDayOfWeek(new Date()) def statMap = CommonDatautils.queryAllDistrictPriceByWeekend(city, weekend) def commids = findAllJnBankDatas(city) - +// def districts = converList(CommonDatautils.findAllDistricts(city, "district", fields)); FzDataUtils.removeNoUpdate(city, districts) districtRun(districts, statMap, commids); @@ -219,14 +219,14 @@ class CZDataBatch extends BaseBatch{ // 商铺名称 map.name = shop.name map.alias = shop.alias - map.build_date = getBuildDate(shop); + map.build_date = FzDataUtils.getBuildDate(shop); //附属属性 map.build_type = shop.build_type map.property_life = shop.property_life map.estate_type2 = shop.estate_type2 map.build_structure = shop.build_structure // 楼盘均价 - map.avg_price = getShopPrice(priceString, shop.city, shop.region, shop.name); + map.avg_price = FzDataUtils.getShopPrice(priceString, shop.city, shop.region, shop.name); // 省 市 区 if(shop.city == "常州"){ map.province_code = regionMap.get("江苏"); @@ -241,28 +241,28 @@ class CZDataBatch extends BaseBatch{ // 地址 map.address = shop.address // 所属区域 - map.in_region_code = getRegionCode(shop.region); + map.in_region_code = FzDataUtils.getRegionCode(shop.region); // 商铺类型 - map.estate_type = getShopEstateTypeCode(shop.estate_type2); + map.estate_type = FzDataUtils.getShopEstateTypeCode(shop.estate_type2); // 公共服务配套 - map.public_service_set_code = getPublicServiceSetCode(shop.city, shop.lat2, shop.lng2); + map.public_service_set_code = FzDataUtils.getPublicServiceSetCode(shop.city, shop.lat2, shop.lng2); // 基础设施 map.facilities_code = "01"; // 商业密度 - map.sophistication_code = getSophisticationCode(shop.city, shop.lat2, shop.lng2); + map.sophistication_code = FzDataUtils.getSophisticationCode(shop.city, shop.lat2, shop.lng2); // 交通条件 - map.traffic_code = getTrafficCode(shop.city, shop.lat2, shop.lng2); + map.traffic_code = FzDataUtils.getTrafficCode(shop.city, shop.lat2, shop.lng2); // 泊车条件 / map.stop_car_code = "01"; // 临街道路 - map.street_code = getStreetCode(street); + map.street_code = FzDataUtils.getStreetCode(street); // 租金 - map.shop_rent = getRentNumber(rentPrice, shop.city, shop.region, shop.name); + map.shop_rent = FzDataUtils.getRentNumber(rentPrice, shop.city, shop.region, shop.name); // 物业公司 - map.estate_company = getEstateCompany(shop.estate_company); + map.estate_company = FzDataUtils.getEstateCompany(shop.estate_company); // 物业费(每月每平米) - map.estate_charge = getEstateCharge(shop.estate_charge); + map.estate_charge = FzDataUtils.getEstateCharge(shop.estate_charge); saveShop(map); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy index da882ff58..0b095093c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy @@ -414,7 +414,7 @@ public class ExcelUtils{ def data = ExcelUtils.read(fileName, srcSheetIndex, 0) def body = data.subList(1, data.size()) - def nThread = Runtime.getRuntime().availableProcessors() * 6 + def nThread = Runtime.getRuntime().availableProcessors() * 8 println nThread ExecutorService executorService = Executors.newFixedThreadPool(nThread) final AtomicInteger counter = new AtomicInteger(0) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy index 071511118..7a14f7431 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy @@ -28,7 +28,10 @@ class GaoDeApiUtils { params.put("location", location) params.put("extensions","all") params.put("radius", "3000") - params.put("poitype", "120000|120200|120203|120300|120301|120302|120303") +// params.put("poitype", "120000|120200|120203|120300|120301|120302|120303")//纯住宅类型 + + //京东测试允许类型 + params.put("poitype", "100000|100100|100101|100102|100103|100104|100105|100200|100201|120000|120200|120203|120300|120301|120302|120303|120304|190000|190100|190400|190401|190402|190403|190500|190600|190700") String content def stop = true JSONObject jsonObj = null @@ -56,6 +59,40 @@ class GaoDeApiUtils { return jsonObj } + def static geocodeByAddress(String city, String address) { + Map params = new HashMap() + params.put("key", GAO_DE_AK) + params.put("address", address) + params.put("batch", "true") + params.put("city", city) +// params.put("poitype", "120000|120200|120203|120300|120301|120302|120303") + String content + def stop = true + JSONObject jsonObj = null + def errorCount = 1 + while (stop){ + try { + content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/geo", params, "utf-8") + } catch (Exception e) { + content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/geo", params, "utf-8") + } + jsonObj = JSONObject.fromObject(content) + println jsonObj + if(jsonObj.status!="1"){ + println jsonObj + errorCount ++ + Thread.sleep(3000) + if(errorCount>3){ + stop = false + } + }else{ + stop = false + } + } + + return jsonObj + } + def static geo(city, address) { Map params = new HashMap() params.put("key", "a34894d151fffff94f9a33bae3d5565c")//许镕杰