From 1e01b4b9d06ac36a6b5d6c22819badddc88311d0 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Mon, 10 Nov 2025 08:33:57 +0800 Subject: [PATCH 01/14] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E9=AB=98=E5=BE=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fuzhou/DistrictSellStatBatch.groovy | 58 +++++++++----- .../com/fangjia/script/gj/OpenApiUtils.groovy | 13 ++++ .../fangjia/script/gj/abc/SuzouPocTest.groovy | 78 ++++++++++++++----- .../gj/data_handle/DevUpdateTestBatch.groovy | 6 +- .../gj/data_handle/NewGaoDePoiBatch.groovy | 2 +- .../gj/data_handle/TownStandardBatch.groovy | 4 +- .../gj/district/DistrictLocationGeo.groovy | 35 +++++++++ .../jiangnanblank/TestEvaluation.groovy | 21 ++++- .../fangjia/script/util/OpenApiClient.groovy | 69 ++++++++++++++++ .../yinjihaun/amap/GaoDeApiUtils.groovy | 4 +- 10 files changed, 242 insertions(+), 48 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictLocationGeo.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy index 7d934e984..4da8b5658 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy @@ -37,14 +37,14 @@ class DistrictSellStatBatch extends BaseBatch { @Override void process(Object args) { -// def months = DateUtils.getMonthList(202206) -// months = months[0..(months.size()-2)] -// println months -// for(def month : months){ -// def curDate = DateUtils.parseFromTightByPattern(month.toString()+"01", "yyyyMMdd") -// calcMonthlyPrice(curDate) -// } - calcMonthlyPrice() + def months = DateUtils.getMonthList(202209) + months = months[0..(months.size()-1)] + println months + for(def month : months){ + def curDate = DateUtils.parseFromTightByPattern(month.toString()+"01", "yyyyMMdd") + calcMonthlyPrice(curDate) + } +// calcMonthlyPrice() } @@ -70,6 +70,8 @@ class DistrictSellStatBatch extends BaseBatch { curDate = new Date() } def citys = CommonUtils.findAllCity() + citys.remove("上海") + citys.remove("福州") def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1).append("status", 1) def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(curDate)) @@ -87,22 +89,25 @@ class DistrictSellStatBatch extends BaseBatch { for(def city : citys){ index++ + if(index < 242){ + continue + } def districts = [] if(city == "福州"){ - if(day == 2){ + if(day == 29){ districts = FZ_DIST_COLL.find(new BasicDBObject("city", city), keys).toList() }else{ districts = FZ_DIST_COLL.find(new BasicDBObject("city", city).append("m_date", new BasicDBObject('$gte',startMDate).append('$lt', endMDate)), keys).toList() } }else{ def q = new BasicDBObject("city", city).append("cat", "district").append("status", 0) -// q.put("name", "翠园楼") - if(day == 2){ +// q.put("name", "越华路小区") +// if(day == 30){ districts = SEAWEED_MONGO_COLL.find(q, keys).batchSize(2000).toList() - }else{ - q.put("m_date", new BasicDBObject('$gte',startMDate).append('$lt', endMDate)) - districts = SEAWEED_MONGO_COLL.find(q, keys).batchSize(2000) - } +// }else{ +// q.put("m_date", new BasicDBObject('$gte',startMDate).append('$lt', endMDate)) +// districts = SEAWEED_MONGO_COLL.find(q, keys).batchSize(2000) +// } } println index+" "+city +" "+districts.size()+" "+ curDate.toString() @@ -155,7 +160,7 @@ class DistrictSellStatBatch extends BaseBatch { def key = dist.region +"##" + dist.name def weekendPriceObj = weekendPriceMap.get(key) - if(weekendPriceObj){ + if(weekendPriceObj && weekendPriceObj.avg_price){ def curPriceObj = curMonthlyPriceMap.get(key) def lastMonthPriceObj = lastMonthlyPriceMap.get(key) def lastYearPriceObj = lastYearPriceMap.get(key) @@ -216,8 +221,19 @@ class DistrictSellStatBatch extends BaseBatch { data.put("credit", weekendPriceObj.credit) def q = new BasicDBObject() - q.put("date", curMonth) - q.put("commid", dist._id.toString()) + if(curPriceObj.commid){ + q.put("date", curMonth) + q.put("commid", dist._id.toString()) + }else{ + q.put("date", curMonth) + q.put("city", dist.city) + q.put("region", dist.region) + q.put("district_name", dist.name) + + data.put("commid", dist._id.toString()) + } + + DISTRICT_SELL_STAT.update(q , new BasicDBObject('$set', data),true,false) // DISTRICT_SELL_STAT.update(new BasicDBObject("_id", curPriceObj._id) , new BasicDBObject('$set', data),true,false) println curMonth+" [update]"+totalCount.get()+" "+ dist.city+" "+dist.name @@ -239,6 +255,8 @@ class DistrictSellStatBatch extends BaseBatch { DISTRICT_SELL_STAT.save(data) println curMonth+" [save]"+totalCount.get()+" "+ dist.city+" "+dist.name } + + } } catch (Exception e) { @@ -281,6 +299,10 @@ class DistrictSellStatBatch extends BaseBatch { if(d.avg_price_2b && d.value_type!=1){ d.avg_price = d.avg_price_2b } + }else{ + if(d.avg_price_2b){ + d.avg_price = d.avg_price_2b + } } priceMap.put(d.region +"##"+d.district_name, d) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy index 6ded33f82..8026c0447 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy @@ -945,4 +945,17 @@ class OpenApiUtils { return null; } + + static def lactionEvaluation(area, lng, lat){ + def url = "http://192.168.0.100:30020/avmWeb/avmAnn?area=${area}&location=lng"+","+lat + String content = new HttpRequest().get(url, "utf-8"); + if(content){ + def doc = new JsonSlurper().parseText(content) + if(doc && doc.msgCode ==200 && doc.result){ + def dtos = doc.result + return dtos + } + } + return null; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/abc/SuzouPocTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/abc/SuzouPocTest.groovy index 4693542de..4b5e0f2e3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/abc/SuzouPocTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/abc/SuzouPocTest.groovy @@ -36,7 +36,8 @@ class SuzouPocTest { // task.newRun() // task.test()//有楼层 // task.testEval()//无楼层 - task.level()//楼盘评级 +// task.level()//楼盘评级 + task.house() } def test(){ @@ -97,36 +98,73 @@ class SuzouPocTest { } def testEval(){ - ExcelUtils.modify("D:\\数据\\其它\\附件1:POC测试样本(匹配).xlsx", { d -> - if(d['O'] && !d['T']){ + ExcelUtils.modify("D:\\数据\\宏信息\\样本房源-估值.xlsx", { d -> + if(!d['O'] && d['E']){ // def city = d['C'] // if(city == "市辖区"){ // city = d['B'] // } - def obj = CommonDatautils.findDistrictById(d['O']) - if(obj){ - d['R'] = obj.city - d['S'] = obj.region - d['T'] = obj.name - def ret1 = OpenApiClient.addressEvaluation(obj.city, obj.region, obj.city+obj.region+obj.name, obj.name, d['F'], null, null) +// def obj = CommonDatautils.findDistrictById(d['O']) +// if(obj){ +// d['R'] = obj.city +// d['S'] = obj.region +// d['T'] = obj.name + def ret1 = OpenApiClient.addressEvaluation(d['C'], d['D'], d['C']+d['D']+d['E'], d['E'], d['I'], null, null) if (ret1) { - d['P'] = ret1.totalPrice - d['Q'] = ret1.avgPrice - }else{ - def dbo = CommonDatautils.queryFirstDistrictPrice(obj.city, obj.region, obj.name) - if(dbo) { - d['P'] = (int)(dbo.avg_price_2b*d['F'].toString().toDouble()) - d['Q'] = dbo.avg_price_2b - - } - + d['J'] = ret1.totalPrice + d['K'] = ret1.avgPrice + d['L'] = ret1.city + d['M'] = ret1.district + d['N'] = ret1.name } - } +// else{ +// def dbo = CommonDatautils.queryFirstDistrictPrice(obj.city, obj.region, obj.name) +// if(dbo) { +// d['P'] = (int)(dbo.avg_price_2b*d['F'].toString().toDouble()) +// d['Q'] = dbo.avg_price_2b +// +// } +// +// } +// } } }); } + def house(){ + def cases = [] + cases << ["城市", "区域","板块", "名称", "面积", "单价","总价","装修", "朝向", "距离","所在楼层","总楼层","室","厅","卫","发布时间","标题", "序号"] + ExcelUtils.modify("D:\\数据\\宏信息\\样本房源-估值.xlsx", { d -> + if(d['L']){ + def obj = CommonDatautils.findDistrict(d['L'], d['M'], d['N'], "district") + if(obj){ + def lng = obj.lng/10000000000.0 + def lat = obj.lat/10000000000.0 + def location = String.valueOf(lng)+","+ String.valueOf(lat) + def result = OpenApiClient.houseAroundBylocationAction(location, d['I'], 0, 0, 50) + if(result){ + for(def h : result){ + cases << [h.city, h.district, h.block, h.name, h.size, h.unitPrice, h.totalPrice, h.decoration, h.direction, h.distance, h.floor,h.maxFloor, h.room, h.hall, h.toilet, h.publishDate, h.title, d["A"],"挂牌"] + } + } + def address = d['L']+d['M']+d['N'] + def dealRets = OpenApiClient.dealHouse(d['L'], d['M'], d['N'], address, d['I']) + if(dealRets){ + for(def h : dealRets){ + cases << [h.city, h.district, "", h.name, h.size, h.avgPrice, h.totalPrice, h.decoration, h.direction, "", h.floor,h.maxFloor, h.room, h.hall, h.toilet, h.tradeDate, "", d["A"], "成交"] + } + } + + } + + } + + }) + println cases.size() + ExcelUtils.write("D:\\数据\\宏信息\\样本房源-估值.xlsx", cases, "样本") + } + def newRun(){ println("start:"+new Date().toString()) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy index 50ec412b8..45332f6f1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy @@ -33,13 +33,11 @@ class DevUpdateTestBatch { static main(args) { def task = new DevUpdateTestBatch() def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx -// task.addDev(path) - task.addDevAddress(path) + task.addDev(path) +// task.addDevAddress(path) // // task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是") - - } def addDev(path){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy index ea28d2c0b..70b407ceb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy @@ -191,7 +191,7 @@ class NewGaoDePoiBatch extends BaseBatch{ if(smallArr && smallArr.size()>=1){ saveObj.small_type = smallArr[0] }else{ - saveObj.small_type = smallstr + saveObj.8 = smallstr } }else{ saveObj.small_type = saveObj.secondary_type diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy index 3797a3924..8edaa8a99 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy @@ -57,9 +57,9 @@ class TownStandardBatch extends BaseBatch{ } def TownStandard(){ - townStandardByCat("office") +// townStandardByCat("office") // townStandardByPoi() -// addTownByTownShape()//为没有乡镇街道的poi,补充乡镇街道,忽略区域,并使用地图 + addTownByTownShape()//为没有乡镇街道的poi,补充乡镇街道,忽略区域,并使用地图 // standardTown() // updateTownData() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictLocationGeo.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictLocationGeo.groovy new file mode 100644 index 000000000..04ec66d5f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictLocationGeo.groovy @@ -0,0 +1,35 @@ +package com.fangjia.script.gj.district + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.bson.types.ObjectId + +class DistrictLocationGeo { + + final DBCollection seaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "seaweed_gaode_pois", "192.168.0.235", 27018) + + static main(args) { + def task = new DistrictLocationGeo() + task.run()//调用高德逆解析,并保存数据 + } + def run(){ + ExcelUtils.modify("D:\\数据\\京东\\需逆解析数据-test.xlsx",{ d-> + def jsonObject = GaoDeApiUtils.geocode(d['C']) + if(jsonObject){ + def dbo = new BasicDBObject() + dbo.put("old_id", d['B']) + dbo.put("old_location", d['C']) + dbo.put("formatted_address", jsonObject.regeocode.formatted_address) + dbo.put("addressComponent", jsonObject.regeocode.addressComponent) + dbo.put("pois", jsonObject.regeocode.pois) + dbo.put("roads", jsonObject.regeocode.roads) + dbo.put("roadinters", jsonObject.regeocode.roadinters) + dbo.put("aois", jsonObject.regeocode.aois) + seaweedGaodePoisColl.save(dbo) + } + }) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/TestEvaluation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/TestEvaluation.groovy index 8621a2787..d4bff37bb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/TestEvaluation.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/TestEvaluation.groovy @@ -20,10 +20,12 @@ class TestEvaluation { // task.match() // task.houseSimilar() - task.newMatch() +// task.newMatch() // task.houseSimilar() - task.newAndOldMatch() +// task.newAndOldMatch() + + task.addFjwToBprice("D:\\数据\\估值\\地址估价-匹配(加写字楼-商铺) - v2.xlsx") } def houseSimilar(){ @@ -206,4 +208,19 @@ class TestEvaluation { }) } + def addFjwToBprice(path){ + ExcelUtils.multiThreadingModify(path,{d-> + def dist = CommonDatautils.findDistrictById(d['B']) + if(dist){ + def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name) + if(dbo){ + if(dbo.avg_price_2b){ + d['H'] = dbo.avg_price_2b + } + } + } + }) + } + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index 9d7fad788..c3dbfe651 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -8,6 +8,7 @@ import com.mongodb.DBCollection import com.mongodb.DBCursor import net.sf.json.JSONArray import net.sf.json.JSONObject +import net.sf.json.groovy.JsonSlurper import org.apache.commons.lang.StringUtils import org.apache.commons.lang.SystemUtils @@ -1577,5 +1578,73 @@ public class OpenApiClient { + public static def houseAroundBylocationAction(location, size, propertyType, houseType, pageSize) { + + def url ="https://open.fangjia.com/house/around?location="+location+"&size="+size+"&pageSize="+pageSize+"&propertyType="+propertyType+"&houseType="+houseType + url = url +"&token="+TOKEN + + def content = new HttpRequest().newGet(url, null, "utf-8") + if(content){ + def doc = new JsonSlurper().parseText(content) + if(doc && doc.code ==200 && doc.result){ + def dtos = doc.result + return dtos + } + } + + return null + } + + + static def dealHouse(city, region, name, area, address){ + + def url ="https://open.fangjia.com/property/deal?city="+city+"&address="+address+"&district="+region+"&name="+name+"&size="+area + url = url +"&token="+TOKEN + + def content = new HttpRequest().newGet(url, null, "utf-8") + if(content){ + def doc = new JsonSlurper().parseText(content) + if(doc && doc.code ==200 && doc.result){ + def dtos = doc.result + return dtos + } + } + + return null + } + + + public static JSONObject propertyHouseAction(city, district, name, size, propertyType) { + try { + Map params = new HashMap(); + params.put("city", city); + params.put("district", district) + params.put("name", name) + params.put("size", size.toString()); + params.put("token", TOKEN); + params.put("propertyType", propertyType) + params.put("__refresh_cache","1") + + String content = new HttpRequest().get("https://open.fangjia.com/property/house", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + while (result == null) { + result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/property/house", params, "utf-8") + println("${city} ${name} 程序调用异常") + Thread.sleep(6000) + } + if (result.getInt("code") == 0) { + return result.getJSONObject("result"); + } else if (result.getInt("code") !=0) { + println("${city} ${name} ") + } + } catch (Exception e) { + println("${city} ${name} ${e.getMessage()}") + e.printStackTrace(); + } + return null + } + + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy index 60dc4ddb2..c9dceabe6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy @@ -13,7 +13,7 @@ import java.util.zip.GZIPInputStream class GaoDeApiUtils { //94c74c60e102972a8d17b7b75d7403be - def static GAO_DE_AK = "94c74c60e102972a8d17b7b75d7403be" //宝剑:c6fcd4bfbc40d73ee762b8115dca3390 + def static GAO_DE_AK = "a34894d151fffff94f9a33bae3d5565c" //宝剑:c6fcd4bfbc40d73ee762b8115dca3390 //15021384995:dfc2245bac8a7b2d5b62e060e59d4eca /** @@ -29,6 +29,8 @@ class GaoDeApiUtils { params.put("key", GAO_DE_AK) params.put("location", location) params.put("extensions","all") + params.put("radius", "3000") + params.put("poitype", "120000|120200|120203|120300|120301|120302|120303") String content def stop = true JSONObject jsonObj = null From 7694a310e24edcd3170c5d7ea84c5139ee43e1a0 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 10 Nov 2025 09:56:06 +0800 Subject: [PATCH 02/14] =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/gj/OpenApiUtils.groovy | 17 ----------------- .../script/gj/village/XiamenVillage.groovy | 2 +- 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy index 6ded33f82..bfc91de49 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy @@ -847,23 +847,6 @@ class OpenApiUtils { } - static def addrVerifyShBank(city, address){ - address = URLEncoder.encode(address, "utf-8"); -// def url = "http://192.168.0.100:32080/FangJiaAddressService/cebEnterpriseVillageIdentify?address=${address}" - def url = "http://localhost:8080/addrSvc/boscEnterpriseVillageIdentify?address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" - if(city){ - url = url + "&city=${city}" - } - String content = new HttpRequest().get(url, "utf-8"); - if(content){ - def doc = new JsonSlurper().parseText(content) - if(doc && doc.code ==200 && doc.result){ - def dtos = doc.result - return dtos - } - } - return null; - } static def addrVerifyBase(city, address){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/XiamenVillage.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/XiamenVillage.groovy index 6d8badbb2..4337e871e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/XiamenVillage.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/XiamenVillage.groovy @@ -10,7 +10,7 @@ class XiamenVillage { } def run(){ - ExcelUtils.multiThreadingModify("D:\\数据\\厦门\\涉农组织类型判定(to估家,2025-09-30)-结果.xlsx",{ d-> + ExcelUtils.multiThreadingModify("D:\\数据\\厦门\\涉农组织类型判定(to估家,2025-10-31)-以此为准.xlsx",{ d-> if(!d['R']){ def dto = OpenApiUtils.addrVerifyXiameng(null, d['B'], 2) From 428fa5420787a12a5acd594b437af318a01bba68 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 10 Nov 2025 15:46:56 +0800 Subject: [PATCH 03/14] =?UTF-8?q?=E6=9F=A5=E6=88=BF=E6=BA=90=E6=97=B6?= =?UTF-8?q?=E9=97=B4=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/gj/district/DistrictLocationGeo.groovy | 8 +++++--- .../FjwBigdataForsaleBatch.groovy | 15 +++++---------- .../script/yinjihaun/amap/GaoDeApiUtils.groovy | 7 +++---- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictLocationGeo.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictLocationGeo.groovy index 04ec66d5f..eefe4d587 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictLocationGeo.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictLocationGeo.groovy @@ -5,7 +5,6 @@ import com.fangjia.script.util.MongoUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection -import org.bson.types.ObjectId class DistrictLocationGeo { @@ -16,9 +15,10 @@ class DistrictLocationGeo { task.run()//调用高德逆解析,并保存数据 } def run(){ - ExcelUtils.modify("D:\\数据\\京东\\需逆解析数据-test.xlsx",{ d-> + ExcelUtils.multiThreadingModify("D:\\数据\\京东\\需逆解析数据.xlsx", { d -> def jsonObject = GaoDeApiUtils.geocode(d['C']) - if(jsonObject){ + def count = 0 + if (jsonObject) { def dbo = new BasicDBObject() dbo.put("old_id", d['B']) dbo.put("old_location", d['C']) @@ -29,7 +29,9 @@ class DistrictLocationGeo { dbo.put("roadinters", jsonObject.regeocode.roadinters) dbo.put("aois", jsonObject.regeocode.aois) seaweedGaodePoisColl.save(dbo) + count = jsonObject.regeocode.pois.size() } + d['D'] = count }) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy index 9d42392f1..c9801ad9e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy @@ -6,17 +6,15 @@ import com.fangjia.script.Watcher import com.fangjia.script.ccbevaluation.CcbCommonUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.ccb.CcbCodeUtils -import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils -import com.fangjia.script.util.FangJiaAreaMapUtils -import com.google.common.hash.* +import com.google.common.hash.BloomFilter +import com.google.common.hash.Funnels import com.mongodb.BasicDBObject import com.mongodb.DBObject import groovy.sql.Sql import org.apache.commons.lang.StringUtils -import org.bson.types.ObjectId import org.elasticsearch.index.query.BoolQueryBuilder import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders @@ -24,15 +22,12 @@ import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.nio.charset.StandardCharsets import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong import java.util.regex.Matcher import java.util.regex.Pattern - - /** * 建行模型估值项目 * 出售房源数据同步 @@ -201,7 +196,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{ } finally { latch.countDown() } - }; + }; }) } latch.await() @@ -492,8 +487,8 @@ class FjwBigdataForsaleBatch extends BaseBatch{ def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){ -// def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd") - def date = DateUtils.addMonth(endDate.toString(), -12, "yyyyMMdd") + def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd") +// def date = DateUtils.addMonth(endDate.toString(), -12, "yyyyMMdd") def startMdate = DateUtils.parseFromTightByPattern(date,"yyyyMMdd")//DateUtils.parseFromTightByPattern("20240519","yyyyMMdd") def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")//DateUtils.parseFromTightByPattern("20240718", "yyyyMMdd") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy index c9dceabe6..071511118 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy @@ -1,19 +1,17 @@ package com.fangjia.script.yinjihaun.amap import com.fangjia.core.util.HttpRequest -import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.thirdPriceCrawl.UserAgents import com.fangjia.script.util.ABuYunUtil import com.fangjia.script.util.JacksonUtil import net.sf.json.JSONObject import org.apache.commons.lang.StringUtils -import org.jsoup.Jsoup import java.util.zip.GZIPInputStream class GaoDeApiUtils { - //94c74c60e102972a8d17b7b75d7403be - def static GAO_DE_AK = "a34894d151fffff94f9a33bae3d5565c" //宝剑:c6fcd4bfbc40d73ee762b8115dca3390 + //67423021cae3b58e5c2ac595a389b036 :临时买的key + def static GAO_DE_AK = "67423021cae3b58e5c2ac595a389b036" //宝剑:c6fcd4bfbc40d73ee762b8115dca3390 //15021384995:dfc2245bac8a7b2d5b62e060e59d4eca /** @@ -39,6 +37,7 @@ class GaoDeApiUtils { try { content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8") } catch (Exception e) { + println location + " 出错:"+content content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8") } jsonObj = JSONObject.fromObject(content) From 321c830768dd48cb0f34be0354f93116b013d216 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Sun, 16 Nov 2025 10:18:22 +0800 Subject: [PATCH 04/14] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E5=AD=98=E5=82=A8=EF=BC=9Aclass=20net.sf.json.JSONNull?= =?UTF-8?q?=E6=97=A0=E6=B3=95=E5=AD=98=E5=82=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/interceptors/AuditInterceptor.java | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/interceptors/AuditInterceptor.java b/projects/fangjia-open/src/main/java/com/fangjia/open/interceptors/AuditInterceptor.java index dc87931f8..cd2da8ea8 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/interceptors/AuditInterceptor.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/interceptors/AuditInterceptor.java @@ -1,6 +1,7 @@ package com.fangjia.open.interceptors; import com.fangjia.core.util.AnalyticsUtil; +import com.fangjia.core.util.JsonUtils; import com.fangjia.dal.mongo.open.model.OpenResource; import com.fangjia.dal.mongo.tool.model.OpenUriLog; import com.fangjia.dal.sql.model.ApiUser; @@ -12,6 +13,8 @@ import com.fangjia.open.model.BasicModel; import com.fangjia.open.model.RestfulModelAndView; import com.fangjia.service.open.OpenResourceService; import com.fangjia.service.open.OpenUriLogService; +import com.fasterxml.jackson.databind.ObjectMapper; +import net.sf.json.JSONNull; import net.sf.json.JSONObject; import net.sf.json.JsonConfig; import net.sf.json.processors.JsonValueProcessor; @@ -22,17 +25,13 @@ import org.sothis.web.mvc.ActionContext; import org.sothis.web.mvc.ActionInvocation; import org.sothis.web.mvc.ActionInvocationException; import org.sothis.web.mvc.Interceptor; - import javax.servlet.http.HttpServletRequest; -import java.math.BigDecimal; import java.text.SimpleDateFormat; -import java.util.Collection; -import java.util.Date; -import java.util.LinkedHashMap; -import java.util.Map; +import java.util.*; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; + /** * API审计日志监控器 * @@ -116,7 +115,6 @@ public class AuditInterceptor implements Interceptor { } return params; } - /** * @param request * @param apiUser @@ -163,7 +161,7 @@ public class AuditInterceptor implements Interceptor { if(isSaveResult(request)){ JSONObject json = JSONObject.fromObject(result, DEFAULT_JSON_CONFIG); json.remove("tid"); - openUriLog.setResMap(json); + openUriLog.setResMap((Map)JsonUtils.convertJsonToJavaObj(json, Map.class)); } } catch (Exception e) { LOGGER.error("", e); From 0e7cbd435a5a87303b8f97adbb9439b1d01d1d02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Sun, 16 Nov 2025 10:35:33 +0800 Subject: [PATCH 05/14] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=8C=B9=E9=85=8D?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E7=A9=BA=E6=8C=87=E9=92=88=E5=BC=82=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/service/address/AddressParserServiceImpl.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java index 6f6590ee0..caf1898b7 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java @@ -339,7 +339,11 @@ public class AddressParserServiceImpl implements AddressParserService { } } } - district = districtService.findById(district.getId(), MATCH_PROPERTY_FIELDS); + if(district == null){ + logger.error("未找到匹配的CPDB数据:" + addr + "ID:" + matchedMate.getId()); + }else{ + district = districtService.findById(district.getId(), MATCH_PROPERTY_FIELDS); + } if(district == null){ logger.error("未找到匹配的CPDB数据:" + addr + "ID:" + district.getId()); } From ece1bd56f405bea388b6d7b9fb4cc43c600feef0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Sun, 16 Nov 2025 10:49:57 +0800 Subject: [PATCH 06/14] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=8C=B9=E9=85=8D?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E7=A9=BA=E6=8C=87=E9=92=88=E5=BC=82=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/service/address/AddressParserServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java index caf1898b7..e49a3286a 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java @@ -345,7 +345,7 @@ public class AddressParserServiceImpl implements AddressParserService { district = districtService.findById(district.getId(), MATCH_PROPERTY_FIELDS); } if(district == null){ - logger.error("未找到匹配的CPDB数据:" + addr + "ID:" + district.getId()); + logger.error("未找到匹配的CPDB数据:" + addr + "ID:" + matchedMate.getId()); } return new Building(district, scoreResult.getScore(matchedMate)); }else{ From 1a15f8d2cff63f361648975f7a4172f089373a58 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Wed, 19 Nov 2025 11:42:41 +0800 Subject: [PATCH 07/14] =?UTF-8?q?=E5=B8=B8=E5=B7=9E=E5=95=86=E9=93=BA?= =?UTF-8?q?=E5=90=8C=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/jiangnanblank/CZDataBatch.groovy | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy index 4e23f59e1..5a037dba9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy @@ -56,7 +56,7 @@ class CZDataBatch extends BaseBatch{ def weekend = DateUtils.getFirstDayOfWeek(new Date()) def statMap = CommonDatautils.queryAllDistrictPriceByWeekend(city, weekend) def commids = findAllJnBankDatas(city) - +// def districts = converList(CommonDatautils.findAllDistricts(city, "district", fields)); FzDataUtils.removeNoUpdate(city, districts) districtRun(districts, statMap, commids); @@ -219,14 +219,14 @@ class CZDataBatch extends BaseBatch{ // 商铺名称 map.name = shop.name map.alias = shop.alias - map.build_date = getBuildDate(shop); + map.build_date = FzDataUtils.getBuildDate(shop); //附属属性 map.build_type = shop.build_type map.property_life = shop.property_life map.estate_type2 = shop.estate_type2 map.build_structure = shop.build_structure // 楼盘均价 - map.avg_price = getShopPrice(priceString, shop.city, shop.region, shop.name); + map.avg_price = FzDataUtils.getShopPrice(priceString, shop.city, shop.region, shop.name); // 省 市 区 if(shop.city == "常州"){ map.province_code = regionMap.get("江苏"); @@ -241,28 +241,28 @@ class CZDataBatch extends BaseBatch{ // 地址 map.address = shop.address // 所属区域 - map.in_region_code = getRegionCode(shop.region); + map.in_region_code = FzDataUtils.getRegionCode(shop.region); // 商铺类型 - map.estate_type = getShopEstateTypeCode(shop.estate_type2); + map.estate_type = FzDataUtils.getShopEstateTypeCode(shop.estate_type2); // 公共服务配套 - map.public_service_set_code = getPublicServiceSetCode(shop.city, shop.lat2, shop.lng2); + map.public_service_set_code = FzDataUtils.getPublicServiceSetCode(shop.city, shop.lat2, shop.lng2); // 基础设施 map.facilities_code = "01"; // 商业密度 - map.sophistication_code = getSophisticationCode(shop.city, shop.lat2, shop.lng2); + map.sophistication_code = FzDataUtils.getSophisticationCode(shop.city, shop.lat2, shop.lng2); // 交通条件 - map.traffic_code = getTrafficCode(shop.city, shop.lat2, shop.lng2); + map.traffic_code = FzDataUtils.getTrafficCode(shop.city, shop.lat2, shop.lng2); // 泊车条件 / map.stop_car_code = "01"; // 临街道路 - map.street_code = getStreetCode(street); + map.street_code = FzDataUtils.getStreetCode(street); // 租金 - map.shop_rent = getRentNumber(rentPrice, shop.city, shop.region, shop.name); + map.shop_rent = FzDataUtils.getRentNumber(rentPrice, shop.city, shop.region, shop.name); // 物业公司 - map.estate_company = getEstateCompany(shop.estate_company); + map.estate_company = FzDataUtils.getEstateCompany(shop.estate_company); // 物业费(每月每平米) - map.estate_charge = getEstateCharge(shop.estate_charge); + map.estate_charge = FzDataUtils.getEstateCharge(shop.estate_charge); saveShop(map); } From 3ffa7fb9571c9e2dca23be726295c01e007f176e Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Wed, 19 Nov 2025 11:45:27 +0800 Subject: [PATCH 08/14] =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/commData/CommonDatautils.groovy | 3 +- .../fuzhou/DistrictInfoExportBatch.groovy | 1 + .../fuzhou/DistrictPriceExportBatch.groovy | 1 + .../com/fangjia/script/gj/OpenApiUtils.groovy | 36 ++--- .../gj/data_handle/DevUpdateTestBatch.groovy | 4 +- .../gj/district/DistrictLocationGeo.groovy | 151 ++++++++++++++++-- .../gj/district/FangjiaClientUtils.groovy | 2 +- .../script/gj/village/KunlunVillage.groovy | 25 ++- .../com/fangjia/script/util/ExcelUtils.groovy | 2 +- .../yinjihaun/amap/GaoDeApiUtils.groovy | 41 ++++- 10 files changed, 224 insertions(+), 42 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index c64ee5c4b..736ff631d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -813,8 +813,9 @@ class CommonDatautils { // q.put("status",new BasicDBObject('$ne', 0)) q.put("cat", cat); q.put("city", city) + // q.put("loop_line_info",new BasicDBObject('$exists',false)) -// q.put("name","经纬天地园二期") +// q.put("name","扬州北街8号") DBObject keys = new BasicDBObject(); if(fields){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy index d49b696f3..9c7fa617d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy @@ -211,6 +211,7 @@ class DistrictInfoExportBatch extends BaseBatch{ def getFzDistricts(def city, def mdate, def endMDate){ def q = new BasicDBObject() q.put("city", city) + if(mdate){ q.put("m_date", new BasicDBObject('$gte', mdate).append('$lt', endMDate)) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy index 271b157aa..862fd5f8f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy @@ -99,6 +99,7 @@ class DistrictPriceExportBatch extends BaseBatch{ def getFzDistricts(def city, def mdate, def endMDate){ def q = new BasicDBObject() q.put("city", city) + if(mdate){ q.put("m_date", new BasicDBObject('$gte', mdate).append('$lt', endMDate)) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy index 8026c0447..cb3b51b5a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy @@ -831,7 +831,7 @@ class OpenApiUtils { static def addrVerifyGuangda(city, address){ address = URLEncoder.encode(address, "utf-8"); def url = "http://192.168.0.100:32080/FangJiaAddressService/cebEnterpriseVillageIdentify?address=${address}" -// def url = "http://localhost:8080/fangjia-open/addrSvc/njcbVillageIdentify?address=${address}&type=${type}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" +// def url = "http://localhost:8080/FangJiaAddressService/cebEnterpriseVillageIdentify?address=${address}" if(city){ url = url + "&city=${city}" } @@ -847,23 +847,23 @@ class OpenApiUtils { } - static def addrVerifyShBank(city, address){ - address = URLEncoder.encode(address, "utf-8"); -// def url = "http://192.168.0.100:32080/FangJiaAddressService/cebEnterpriseVillageIdentify?address=${address}" - def url = "http://localhost:8080/addrSvc/boscEnterpriseVillageIdentify?address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" - if(city){ - url = url + "&city=${city}" - } - String content = new HttpRequest().get(url, "utf-8"); - if(content){ - def doc = new JsonSlurper().parseText(content) - if(doc && doc.code ==200 && doc.result){ - def dtos = doc.result - return dtos - } - } - return null; - } +// static def addrVerifyShBank(city, address){ +// address = URLEncoder.encode(address, "utf-8"); +//// def url = "http://192.168.0.100:32080/FangJiaAddressService/cebEnterpriseVillageIdentify?address=${address}" +// def url = "http://localhost:8080/addrSvc/boscEnterpriseVillageIdentify?address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" +// if(city){ +// url = url + "&city=${city}" +// } +// String content = new HttpRequest().get(url, "utf-8"); +// if(content){ +// def doc = new JsonSlurper().parseText(content) +// if(doc && doc.code ==200 && doc.result){ +// def dtos = doc.result +// return dtos +// } +// } +// return null; +// } static def addrVerifyBase(city, address){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy index 45332f6f1..6650ea864 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy @@ -33,8 +33,8 @@ class DevUpdateTestBatch { static main(args) { def task = new DevUpdateTestBatch() def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx - task.addDev(path) -// task.addDevAddress(path) +// task.addDev(path) + task.addDevAddress(path) // // task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictLocationGeo.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictLocationGeo.groovy index 04ec66d5f..be6dc0321 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictLocationGeo.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictLocationGeo.groovy @@ -3,32 +3,157 @@ package com.fangjia.script.gj.district import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.BasicDBList import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import net.sf.json.JSONNull import org.bson.types.ObjectId class DistrictLocationGeo { final DBCollection seaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "seaweed_gaode_pois", "192.168.0.235", 27018) + final DBCollection allSeaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "all_seaweed_gaode_pois", "192.168.0.235", 27018) static main(args) { def task = new DistrictLocationGeo() - task.run()//调用高德逆解析,并保存数据 + task.run()//通过经纬度,调用高德逆解析,并保存数据 +// task.run2();//通过地址,调用高德逆解析 } def run(){ - ExcelUtils.modify("D:\\数据\\京东\\需逆解析数据-test.xlsx",{ d-> - def jsonObject = GaoDeApiUtils.geocode(d['C']) + ExcelUtils.modify("D:\\数据\\京东\\失败.xlsx",{ d-> + if(d['E']){ + def jsonObject = GaoDeApiUtils.geocode(d['E']) + if(jsonObject){ + d["F"] = "高德" + def dbo = new BasicDBObject() + dbo.put("old_id", d['A']) + dbo.put("city", d['B']) + dbo.put("region", d['C']) + dbo.put("name", d['D']) + dbo.put("old_location", d['E']) + dbo.put("formatted_address", jsonObject.regeocode.formatted_address) + def addrComp = jsonObject.regeocode.addressComponent +// addrComp.remove("neighborhood") +// addrComp.remove("building") + dbo.put("addressComponent", addrComp) + dbo.put("pois", jsonObject.regeocode.pois) + dbo.put("roads", jsonObject.regeocode.roads) + dbo.put("roadinters", jsonObject.regeocode.roadinters) + dbo.put("aois", jsonObject.regeocode.aois) + dbo = cleanGeoData(dbo) + dbo = convertToDBObject(dbo) + println(dbo) + + allSeaweedGaodePoisColl.save(dbo) + } + } + }) + + + } + + static BasicDBObject convertToDBObject(Map map) { + if (!map) return null + + BasicDBObject dbObject = new BasicDBObject() + map.each { key, value -> + def convertedValue = convertValue(value) + if (convertedValue != null) { + dbObject.put(key, convertedValue) + } + } + return dbObject + } + + static Object convertValue(Object value) { + if (value == null) return null + + switch (value) { + case Map: + return convertToDBObject(value as Map) + case List: + return convertList(value as List) + default: + return value + } + } + + static BasicDBList convertList(List list) { + BasicDBList dbList = new BasicDBList() + list.each { item -> + def convertedItem = convertValue(item) + if (convertedItem != null) { + dbList.add(convertedItem) + } + } + return dbList + } + + static def cleanGeoData(data) { + if (data == null) return null + + def result = [:] + + data.each { key, value -> + def cleanedValue = cleanValue(value) + if (cleanedValue != null && !isEmptyCollection(cleanedValue)) { + result[key] = cleanedValue + } + } + + return result + } + + static def cleanValue(value) { + if (value == null) return null + + switch (value) { + case Map: + def cleanedMap = [:] + value.each { k, v -> + def cleaned = cleanValue(v) + if (cleaned != null && !isEmptyCollection(cleaned)) { + cleanedMap[k] = cleaned + } + } + return cleanedMap ? cleanedMap : null + + case List: + def cleanedList = value.collect { cleanValue(it) } + .findAll { it != null && !isEmptyCollection(it) } + return cleanedList ? cleanedList : null + + case JSONNull: + return null + + default: + // 检查是否是包含JSONNull的字符串表示 + if (value instanceof String && value.contains("JSONNull")) { + return null + } + return value + } + } + + static def isEmptyCollection(obj) { + if (obj instanceof Map) return obj.isEmpty() + if (obj instanceof List) return obj.isEmpty() + return false + } + + def run2(){ + ExcelUtils.modify("D:\\数据\\昆仑\\263未出居委数据.xlsx",{ d-> + def jsonObject = GaoDeApiUtils.geocodeByAddress(d['C'], d['B']) + Thread.sleep(1000) if(jsonObject){ - def dbo = new BasicDBObject() - dbo.put("old_id", d['B']) - dbo.put("old_location", d['C']) - dbo.put("formatted_address", jsonObject.regeocode.formatted_address) - dbo.put("addressComponent", jsonObject.regeocode.addressComponent) - dbo.put("pois", jsonObject.regeocode.pois) - dbo.put("roads", jsonObject.regeocode.roads) - dbo.put("roadinters", jsonObject.regeocode.roadinters) - dbo.put("aois", jsonObject.regeocode.aois) - seaweedGaodePoisColl.save(dbo) + def rets = jsonObject.geocodes + if(rets){ + def dbo = rets[0] + if(dbo){ + d['D'] = dbo.formatted_address + d['E'] = dbo.level + } + } } }) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index ad2d5309a..0dfe2daab 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -355,7 +355,7 @@ class FangjiaClientUtils { address = address.toString().replace("{","") int retryCount = 0 while (retryCount < 3) { - url = "http://192.168.0.93:8080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true&fields=dev_confi" + url = "http://192.168.0.100:32080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true&fields=dev_confi" if(city){ url = url +"&city="+URLEncoder.encode(city,"UTF-8") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/KunlunVillage.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/KunlunVillage.groovy index d8b140529..86bb2ea86 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/KunlunVillage.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/KunlunVillage.groovy @@ -1,19 +1,36 @@ package com.fangjia.script.gj.village import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.util.ExcelUtils class KunlunVillage { static main(args) { def task = new KunlunVillage(); task.run() +// task.addrCheck() } - def run(){ - ExcelUtils.multiThreadingModify("D:\\数据\\昆仑\\3987户需批量更新涉农信息明细-20251023-结果.xlsx",{ d-> + def addrCheck(){ - if(!d['R'] && d['B']){ - def dto = OpenApiUtils.addrVerifyGuangda(null, d['B']) + ExcelUtils.modify("D:\\数据\\昆仑\\2025年11月18日批次跑批涉农地址-发估家.xlsx",{ d-> + def obj = FangjiaClientUtils.formatNinelevel(null, d["C"]) + if(obj){ + println obj + d['AB'] = obj.addressAssessment.msg + } + }) + + } + + + def run(){ + def index = 0 + ExcelUtils.modify("D:\\数据\\昆仑\\2025年11月18日批次跑批涉农地址-发估家.xlsx",{ d-> + index++ + if(!d['R'] && d['C']){ + println index + def dto = OpenApiUtils.addrVerifyBase(null, d['C']) if(dto){ // def dto = jsonObj.result d['F'] = dto.province diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy index da882ff58..0b095093c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy @@ -414,7 +414,7 @@ public class ExcelUtils{ def data = ExcelUtils.read(fileName, srcSheetIndex, 0) def body = data.subList(1, data.size()) - def nThread = Runtime.getRuntime().availableProcessors() * 6 + def nThread = Runtime.getRuntime().availableProcessors() * 8 println nThread ExecutorService executorService = Executors.newFixedThreadPool(nThread) final AtomicInteger counter = new AtomicInteger(0) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy index c9dceabe6..bf3d8acab 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy @@ -13,7 +13,7 @@ import java.util.zip.GZIPInputStream class GaoDeApiUtils { //94c74c60e102972a8d17b7b75d7403be - def static GAO_DE_AK = "a34894d151fffff94f9a33bae3d5565c" //宝剑:c6fcd4bfbc40d73ee762b8115dca3390 + def static GAO_DE_AK = "947cd51c2f3110a93cbd755f697a526d" //宝剑:c6fcd4bfbc40d73ee762b8115dca3390 //15021384995:dfc2245bac8a7b2d5b62e060e59d4eca /** @@ -30,7 +30,10 @@ class GaoDeApiUtils { params.put("location", location) params.put("extensions","all") params.put("radius", "3000") - params.put("poitype", "120000|120200|120203|120300|120301|120302|120303") +// params.put("poitype", "120000|120200|120203|120300|120301|120302|120303")//纯住宅类型 + + //京东测试允许类型 + params.put("poitype", "100000|100100|100101|100102|100103|100104|100105|100200|100201|120000|120200|120203|120300|120301|120302|120303|120304|190000|190100|190400|190401|190402|190403|190500|190600|190700") String content def stop = true JSONObject jsonObj = null @@ -57,6 +60,40 @@ class GaoDeApiUtils { return jsonObj } + def static geocodeByAddress(String city, String address) { + Map params = new HashMap() + params.put("key", GAO_DE_AK) + params.put("address", address) + params.put("batch", "true") + params.put("city", city) +// params.put("poitype", "120000|120200|120203|120300|120301|120302|120303") + String content + def stop = true + JSONObject jsonObj = null + def errorCount = 1 + while (stop){ + try { + content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/geo", params, "utf-8") + } catch (Exception e) { + content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/geo", params, "utf-8") + } + jsonObj = JSONObject.fromObject(content) + println jsonObj + if(jsonObj.status!="1"){ + println jsonObj + errorCount ++ + Thread.sleep(3000) + if(errorCount>3){ + stop = false + } + }else{ + stop = false + } + } + + return jsonObj + } + def static geo(city, address) { Map params = new HashMap() params.put("key", "a34894d151fffff94f9a33bae3d5565c")//许镕杰 From 88cd67a3fe43aeb9b63612b443a01178b0b2b72b Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Wed, 19 Nov 2025 11:49:40 +0800 Subject: [PATCH 09/14] update --- .../fangjia/script/gj/district/DistrictLocationGeo.groovy | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictLocationGeo.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictLocationGeo.groovy index 999e954a2..512e54ee4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictLocationGeo.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictLocationGeo.groovy @@ -6,14 +6,11 @@ import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBList import com.mongodb.BasicDBObject import com.mongodb.DBCollection -<<<<<<< .mine + import net.sf.json.JSONNull import org.bson.types.ObjectId -======= ->>>>>>> .theirs - class DistrictLocationGeo { final DBCollection seaweedGaodePoisColl = MongoUtils.getCollectionByName("fangjia", "seaweed_gaode_pois", "192.168.0.235", 27018) From d864488e21e548d1a41dfb96c1589034d298fe98 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 24 Nov 2025 11:04:19 +0800 Subject: [PATCH 10/14] =?UTF-8?q?=E6=B1=9F=E5=8D=97=E8=A1=8C=E5=95=86?= =?UTF-8?q?=E9=93=BA=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/commData/CommonDatautils.groovy | 64 +++++++++++++++++++ .../script/jiangnanblank/CZDataBatch.groovy | 55 ++++++++++++---- .../script/jiangnanblank/FzDataUtils.groovy | 1 + 3 files changed, 108 insertions(+), 12 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 736ff631d..ad3537664 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -5818,4 +5818,68 @@ class CommonDatautils { } return address; } + + + static def findSeawater(city, lng, lat, distance, p_date, cat){ + def header = [:] + header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:goojia123456".getBytes("UTF-8"))}".toString() + String seawaterUrl = "http://192.168.0.206:9200/seawater-*/_search" + def body = """ + { + "query": { + "bool": { + "must": [ + { + "geo_distance": { + "distance": "1km", + "location": { + "lat": ${lat}, + "lon": ${lng} + } + } + }, + { + "term": { + "city": "${city}" + } + }, + { + "term": { + "cat": "${cat}" + } + }, + { + "range": { + "publish_date": { + "gte": "${p_date}", + "format": "yyyy-MM-dd" + } + } + } + ] + } + }, + "sort": [ + { + "publish_date": { + "order": "desc" + } + } + ], + "size":30 + } + """ + HttpRequest request = new HttpRequest() + String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8") + def response = new JsonSlurper().parseText(res) + + def esObj = null + if(response && response.hits){ + esObj = response.hits.hits + } + return esObj + } + + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy index 5a037dba9..4f61afd3a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.jiangnanblank +import com.fangjia.core.util.DateUtil import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher @@ -7,24 +8,16 @@ import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils -import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject -import com.rabbitmq.client.AMQP -import net.sf.json.JSONObject import org.apache.poi.hssf.usermodel.HSSFWorkbook import org.bson.types.ObjectId -import java.text.DecimalFormat import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors -import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicLong -import java.util.regex.Matcher -import java.util.regex.Pattern - /** * Created by gaojuan on 2017/12/11. * 批量全国小区及商铺字段转换 @@ -33,7 +26,7 @@ import java.util.regex.Pattern class CZDataBatch extends BaseBatch{ public static def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017"; - def fields = ["city","region","name", "address", "cat","lat2","lng2","alias","property_life","estate_type2","build_date","build_structure","build_type","finish_building_date","live_date","open_date","virescence_percent","cubage_percent","estate_company","estate_charge","car_site", "build_type"] + def fields = ["city","region","name", "address", "cat","lat2","lng2","lat","lng","alias","property_life","estate_type2","build_date","build_structure","build_type","finish_building_date","live_date","open_date","virescence_percent","cubage_percent","estate_company","estate_charge","car_site", "build_type"] public static DBCollection DIST_COLL = MongoUtils.getCollectionByName2("business", "jn_bank_district", dbUrl) //final def czIdMap = readCZDistrict("d:\\常州数据结果\\原始数据\\常州江南银行住宅编号.xls"); @@ -227,11 +220,14 @@ class CZDataBatch extends BaseBatch{ map.build_structure = shop.build_structure // 楼盘均价 map.avg_price = FzDataUtils.getShopPrice(priceString, shop.city, shop.region, shop.name); + if(!map.avg_price){ + map.avg_price = getSellPrice(shop.city, shop.lng/10000000000.0, shop.lat/10000000000.0) + } // 省 市 区 if(shop.city == "常州"){ - map.province_code = regionMap.get("江苏"); - map.city_code = regionMap.get(shop.city); - map.region_code = regionMap.get(shop.region); + map.province_code = FzDataUtils.regionMap.get("江苏"); + map.city_code = FzDataUtils.regionMap.get(shop.city); + map.region_code = FzDataUtils.regionMap.get(shop.region); map.province = "江苏"; map.price_old_string = priceString; map.heights = heights; @@ -259,6 +255,9 @@ class CZDataBatch extends BaseBatch{ map.street_code = FzDataUtils.getStreetCode(street); // 租金 map.shop_rent = FzDataUtils.getRentNumber(rentPrice, shop.city, shop.region, shop.name); + if(!map.shop_rent){ + map.shop_rent = getRentPrice(shop.city, shop.lng/10000000000.0, shop.lat/10000000000.0) + } // 物业公司 map.estate_company = FzDataUtils.getEstateCompany(shop.estate_company); // 物业费(每月每平米) @@ -267,6 +266,38 @@ class CZDataBatch extends BaseBatch{ saveShop(map); } + def getSellPrice(def city, def lng, def lat){ + def p_date = DateUtils.formatDateByPattern(DateUtil.add(new Date(), -365),'yyyy-MM-dd') + def rets = CommonDatautils.findSeawater(city, lng, lat, "1km", p_date, "shop_sell") + if(rets){ + def prices = [] + for(def r : rets){ + prices << r._source.avg_price + } + if(prices){ + def price = (int)(prices.sum()/prices.size()) + return price + } + } + return null; + } + + def getRentPrice(def city, def lng, def lat){ + def p_date = DateUtils.formatDateByPattern(DateUtil.add(new Date(), -365),'yyyy-MM-dd') + def rets = CommonDatautils.findSeawater(city, lng, lat, "1km", p_date, "shop_rent") + if(rets){ + def prices = [] + for(def r : rets){ + prices << r._source.avg_price + } + if(prices){ + def day_price = (prices.sum()/prices.size())/30 + return DataHandleUtils.getDoubleRound(2, day_price) + } + } + return null; + } + def saveShop(def m){ DBObject q = new BasicDBObject(); q.put("city",m.city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/FzDataUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/FzDataUtils.groovy index 51e58e28e..0c5917dfd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/FzDataUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/FzDataUtils.groovy @@ -809,4 +809,5 @@ class FzDataUtils { } + } From 697771b4b0903327fff598b45601b5a4a0774ffd Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 24 Nov 2025 11:47:32 +0800 Subject: [PATCH 11/14] update --- .../house/DistrictSellHouseStatBatch.groovy | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/house/DistrictSellHouseStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/house/DistrictSellHouseStatBatch.groovy index 52203c79c..5742b35c4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/house/DistrictSellHouseStatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/house/DistrictSellHouseStatBatch.groovy @@ -1,25 +1,19 @@ package com.fangjia.script.house -import com.fangjia.core.util.DateUtil + import com.fangjia.datastore.driver.DSCollection import com.fangjia.datastore.driver.DataStore import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils -import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject -import com.mongodb.Bytes import com.mongodb.DBCollection import com.mongodb.DBObject -import com.mongodb.QueryOperators - - import java.util.concurrent.ExecutorService import java.util.concurrent.Executors -import java.util.concurrent.TimeUnit @BatchOwner(Watcher.GaoJuan) class DistrictSellHouseStatBatch extends BaseBatch{ @@ -48,14 +42,17 @@ class DistrictSellHouseStatBatch extends BaseBatch{ for(def city : citys){ index ++ ; def districts = CommonDatautils.getAllDistrictBaseInfo(city, "district") - def sellRet = houseStatCount(districts, startDate, endDate, sDate); + + String start = DateUtils.formatDateByPattern(startDate, "yyyyMMdd") + + def sellRet = houseStatCount(districts, start.toInteger(), sDate); saveResult(city, sDate, sellRet); println "[${city} ${index}/${citys.size()}]" +new Date().toString(); } } - def houseStatCount(def districts, def startDate, def endDate, def sDate){ + def houseStatCount(def districts, def startDate, def sDate){ def index = 0; @@ -75,8 +72,12 @@ class DistrictSellHouseStatBatch extends BaseBatch{ void run() { index++; def map = [:]; - map.sell_count = findHouseCount(dist, "sell", startDate, endDate); - map.rent_count = findHouseCount(dist, "rent", startDate, endDate); +// map.sell_count = findHouseCount(dist, "sell", startDate, endDate); +// map.rent_count = findHouseCount(dist, "rent", startDate, endDate); + + map.sell_count = CommonDatautils.findHouseCountByDate(dist.city, dist.region, dist.name, dist.alias, "sell", startDate) + map.rent_count = CommonDatautils.findHouseCountByDate(dist.city, dist.region, dist.name, dist.alias, "rent", startDate) + map.city = dist.city; map.region = dist.region; map.name = dist.name; From fca72b79670dd19a049a237014ebc299443cd0d2 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 24 Nov 2025 13:17:39 +0800 Subject: [PATCH 12/14] update --- .../com/fangjia/script/house/DistrictSellHouseStatBatch.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/house/DistrictSellHouseStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/house/DistrictSellHouseStatBatch.groovy index 5742b35c4..7f3f4027b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/house/DistrictSellHouseStatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/house/DistrictSellHouseStatBatch.groovy @@ -14,6 +14,7 @@ import com.mongodb.DBObject import java.util.concurrent.ExecutorService import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit @BatchOwner(Watcher.GaoJuan) class DistrictSellHouseStatBatch extends BaseBatch{ From 1f3107504144b7d155235f095369fa866677bedd Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 24 Nov 2025 13:29:54 +0800 Subject: [PATCH 13/14] =?UTF-8?q?=E5=95=86=E9=93=BA=E7=A7=9F=E9=87=91?= =?UTF-8?q?=E6=89=A9=E5=B1=95=E5=88=B02=E5=85=AC=E9=87=8C=EF=BC=8C3?= =?UTF-8?q?=E5=85=AC=E9=87=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/commData/CommonDatautils.groovy | 2 +- .../com/fangjia/script/jiangnanblank/CZDataBatch.groovy | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index ad3537664..8eedfcae0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -5831,7 +5831,7 @@ class CommonDatautils { "must": [ { "geo_distance": { - "distance": "1km", + "distance": "${distance}", "location": { "lat": ${lat}, "lon": ${lng} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy index 4f61afd3a..211a4ccb5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy @@ -285,6 +285,12 @@ class CZDataBatch extends BaseBatch{ def getRentPrice(def city, def lng, def lat){ def p_date = DateUtils.formatDateByPattern(DateUtil.add(new Date(), -365),'yyyy-MM-dd') def rets = CommonDatautils.findSeawater(city, lng, lat, "1km", p_date, "shop_rent") + if(!rets){ + rets = CommonDatautils.findSeawater(city, lng, lat, "2km", p_date, "shop_rent") + if(!rets){ + rets = CommonDatautils.findSeawater(city, lng, lat, "3km", p_date, "shop_rent") + } + } if(rets){ def prices = [] for(def r : rets){ From dec254bd4b3686e8bc7f50734cac7a1c8a635f68 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 24 Nov 2025 14:26:25 +0800 Subject: [PATCH 14/14] =?UTF-8?q?=E6=B1=9F=E5=8D=97=E8=A1=8C=E4=BF=AE?= =?UTF-8?q?=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/commData/CommonDatautils.groovy | 27 +++++++++++++++++++ .../script/jiangnanblank/CZDataBatch.groovy | 20 +++++++++++--- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 8eedfcae0..76eed7432 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -832,6 +832,33 @@ class CommonDatautils { } + public static def findNewUpdateDistricts(def city, def cat, def fields){ + + def startDate = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd") + DBObject q = new BasicDBObject(); + q.put("status",0) +// q.put("status",new BasicDBObject('$ne', 0)) + q.put("cat", cat); + q.put("city", city) + q.put("m_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate))) +// q.put("loop_line_info",new BasicDBObject('$exists',false)) +// q.put("name","扬州北街8号") + + DBObject keys = new BasicDBObject(); + if(fields){ + for(def field : fields){ + keys.put(field, 1); + } + } + def dbsour = SEAWEED_COLL.find(q, keys).batchSize(5000) + def rets = [] + for(def d : dbsour){ + rets << d + } + return rets; + } + + public static def findAllRegionDistricts(def city, def region, def cat, def fields){ DBObject q = new BasicDBObject(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy index 211a4ccb5..5ca528cfb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy @@ -17,6 +17,7 @@ import org.bson.types.ObjectId import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicLong /** * Created by gaojuan on 2017/12/11. @@ -40,6 +41,8 @@ class CZDataBatch extends BaseBatch{ @Override void process(Object args) { + println "args 参数:"+args.toString() + def citys = CommonDatautils.findAllCity(); citys.remove("常州"); @@ -50,12 +53,21 @@ class CZDataBatch extends BaseBatch{ def statMap = CommonDatautils.queryAllDistrictPriceByWeekend(city, weekend) def commids = findAllJnBankDatas(city) // - def districts = converList(CommonDatautils.findAllDistricts(city, "district", fields)); - FzDataUtils.removeNoUpdate(city, districts) + def districts = null; + def shops = null; + if(args && args[0]== "all"){ + districts = converList(CommonDatautils.findAllDistricts(city, "district", fields)); + FzDataUtils.removeNoUpdate(city, districts) + }else{ + districts = converList(CommonDatautils.findNewUpdateDistricts(city, "district", fields)); + } districtRun(districts, statMap, commids); - - def shops = CommonDatautils.findAllDistricts(city, "shop", fields); + if(args && args[0]== "all"){ + shops = CommonDatautils.findAllDistricts(city, "shop", fields); + }else{ + shops = CommonDatautils.findNewUpdateDistricts(city, "shop", fields); + } shops = mergeShop(shops); shopRun(shops); }