From a5e2033e1b6ef2b5807bb84aa82b6802676129c5 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 30 Oct 2023 16:30:53 +0800 Subject: [PATCH] =?UTF-8?q?=E8=84=9A=E6=9C=AC=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/CcbCommonUtils.groovy | 20 ++ .../DealAndThirdPriceDataProcess.groovy | 4 +- .../ccbevaluation/GuiDingPriceBatch.groovy | 2 +- .../fangjia/script/ccbevaluation/Test.groovy | 48 ++-- .../fangjia/script/chongqing/PocTest.groovy | 5 +- .../script/commData/CommonDatautils.groovy | 151 +++++++++--- .../fangjia/script/gj/AddAvgpriceBatch.groovy | 14 +- .../gj/district/FangjiaClientUtils.groovy | 24 +- .../script/gj/meishubao/MsbPocTest.groovy | 227 ++++++++++++++++++ .../com/fangjia/script/gj/test/PoiTest.groovy | 80 ++++++ .../script/gj/xiamen/SeaweedDataSynch.groovy | 54 ++++- .../com/fangjia/script/util/ExcelUtils.groovy | 3 +- 12 files changed, 557 insertions(+), 75 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meishubao/MsbPocTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/PoiTest.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 7546a13d7..3929090a1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -679,4 +679,24 @@ class CcbCommonUtils { return ids; } + + static def findAvgPrice(def commid, def cityid){ + def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1" + def map = [:] + pgsqlInstance.eachRow(sql, { r -> + map.avg_price = r.gujia_bds_clc_avprc + map.type = r.gujia_bds_clmtd + map.month = r.gujiabds_clc_yrmo_val + map.credit_value = r.gujia_bds_cfdc_dgr + if(r.gujia_bds_clmtd == 1){ + map.calc_type = "人工估值" + }else if(r.gujia_bds_clmtd == 2){ + map.calc_type = "样本计算" + }else if(r.gujia_bds_clmtd == 3){ + map.calc_type = "神经网络" + } + }) + return map + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index 641263d9e..1c503c5d0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -392,12 +392,12 @@ class DealPriceDataProcess { def deleteThirdPartyData(){ def citys = CcbCommonUtils.citys_all - def month = 202308 + def month = 202309 def rows = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def set = new HashSet() for(def city : citys) { - def rets = findPeGteThirdParty2(city, 202309) + def rets = findPeGteThirdParty2(city, 202310) def total = 0 println city +" "+rets.size() if(rets){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 67a53ebc7..cf169cc97 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2023-09-27", "2023-10-24") + def prices = CcbCommonUtils.findPersonPrice(city, "2023-09-27", "2023-10-27") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 3f069821a..9adec0396 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -1288,30 +1288,30 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ -def fileName = "D:\\数据\\建行\\房产估值\\指导价\\9.25人工指导价.xlsx" +//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\9.25人工指导价.xlsx" -ExcelUtils.multiThreadingModify(fileName, {d-> - if(d['A']){ -// def region = RegionData.formatRegion(d['B'], d['C']) -// if(region){ -// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district") +//ExcelUtils.multiThreadingModify(fileName, {d-> +// if(d['A']){ +//// def region = RegionData.formatRegion(d['B'], d['C']) +//// if(region){ +//// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district") +//// if(dist){ +//// d['G'] = dist.region +//// d['H'] = dist._id.toString() +//// } +//// } +// if(!d['K']){ +// def dist = CommonDatautils.findDistrictById(d['E']) // if(dist){ -// d['G'] = dist.region -// d['H'] = dist._id.toString() +// d['K'] = dist.region +//// d['I'] = dist.name +//// d['J'] = dist.status +//// def val = d['F'].toString().toDouble().toInteger() + RandomUtil.random(-100,500) +//// d['K'] = val.toInteger() // } // } - if(!d['K']){ - def dist = CommonDatautils.findDistrictById(d['E']) - if(dist){ - d['K'] = dist.region -// d['I'] = dist.name -// d['J'] = dist.status -// def val = d['F'].toString().toDouble().toInteger() + RandomUtil.random(-100,500) -// d['K'] = val.toInteger() - } - } - } -},0) +// } +//},0) @@ -1415,6 +1415,14 @@ def findSampleRate(pgsqlInstance, cityName, yearmonth){ +ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\23号以后新增人工指导价.xlsx", {d-> + def obj = CommonDatautils.findDistrict(d['A'],d['B'],d['C'], "district") + d['H'] = obj._id.toString() + +}) + + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocTest.groovy index a596bbb54..180d2b09f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocTest.groovy @@ -19,10 +19,12 @@ class PocTest { static main(args) { def task = new PocTest(); - task.run(); +// task.run(); // task.export() +// task.distance() } + def export(){ ExcelUtils.modify("D:\\数据\\其它\\广州.xlsx", { row -> def dist = CommonDatautils.findDistrictById(row["B"]) @@ -156,7 +158,6 @@ class PocTest { if(commid){ def dist = CommonDatautils.findDistrictById(commid.trim()) if(dist){ -// row['Q'] = dist.name def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name) if(priceObj){ row['R'] = priceObj.avg_price diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 0ce543b28..509887495 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -87,6 +87,7 @@ class CommonDatautils { public static final DBCollection REGION_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "region_info", MongoUtils.MongoDB_Domain_mongo); public static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com") + public static final DBCollection confirm_price_date = MongoUtils.getCollectionByName("fangjia", "confirm_price_data", "mongo.db.fangjia.com") static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"] @@ -250,10 +251,10 @@ class CommonDatautils { List dbs = cursor.getResults(); return dbs; } - - public static def findAllDistricts(def city, def cat, def fields){ + + public static def findAllDistrictsFromMg(def city, def cat, def fields){ DBObject q = new BasicDBObject(); - q.put("visible", 0); + q.put("status", 0); q.put("cat", cat); q.put("city", city); // q.put("name","文华楼") @@ -265,7 +266,7 @@ class CommonDatautils { } } - DSCursor cursor = getDataStoreConnection().find(q, keys).batchSize(2000) + DSCursor cursor = DbUtils.seaweed.find(q, keys).batchSize(2000) List rets = [] for(def obj : cursor){ rets << obj @@ -793,6 +794,29 @@ class CommonDatautils { } + + public static def findAllDistricts(def city, def cat, def fields){ + + DBObject q = new BasicDBObject(); + q.put("status",0) + q.put("cat", cat); + q.put("city", city) + + DBObject keys = new BasicDBObject(); + if(fields){ + for(def field : fields){ + keys.put(field, 1); + } + } + def dbsour = DbUtils.seaweed.find(q, keys).batchSize(2000) + def rets = [] + for(def d : dbsour){ + rets << d + } + return rets; + } + + public static def findAllDistricMaptByCity(def city, def cat, def fields){ DBObject q = new BasicDBObject(); @@ -2628,6 +2652,7 @@ class CommonDatautils { q.put("cat", cat); q.put('lat2', new BasicDBObject(QueryOperators.LTE, maxLat).append(QueryOperators.GTE, minLat)); q.put('lng2', new BasicDBObject(QueryOperators.LTE, maxLng).append(QueryOperators.GTE, minLng)); + q.put("visible", 0) def cur = getDataStoreConnection().find(q); def ret = []; for(def obj : cur){ @@ -3886,42 +3911,6 @@ class CommonDatautils { return avg_price } - static def findThirdPartyPrice(def city, def region, def name, def startDate, def endDate){ - def q = new BasicDBObject() - q.put("city", city) - q.put("region", region) - q.put("name", name) - q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) - def dbcursor = thirdpartColl.find(q).sort(new BasicDBObject("s_date", -1)) - def map = [:] - for(def db : dbcursor){ - def key = db.city+"##"+db.region+"##"+db.name - def dataMap = map.get(key) - if(!dataMap){ - dataMap = [:] - map.put(key, dataMap) - } - - if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){ - dataMap.put("anjuke_esf_price", db.anjuke_esf_price) - } - if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){ - dataMap.put("anjuke_esf_price", db.get("58tc_esf_price")) - } - - if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){ - dataMap.put("lianjia_esf_price", db.lianjia_esf_price) - } - if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){ - dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price) - } - - if(!dataMap.containsKey("lianjia_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){ - dataMap.put("lianjia_esf_price", db.beike_esf_price) - } - } - return map - } /** @@ -4149,4 +4138,86 @@ class CommonDatautils { } + /** + * 获取小区周边一公里小区数量 + * @param city + * @param lng + * @param lat + * @return + */ + static getDistrictArround(city, lng, lat) { + try { + def query = new BasicDBObject() + query.append("cat", "district") + query.append("city", city) + query.append("visible", 0) + DBObject sdq = new BasicDBObject(); + sdq.put("sd.point", lat + "," + lng); + sdq.put("sd.sfield", "lat_lng"); + sdq.put("sd.distance", String.valueOf((double) 1000 / 1000.0)); + query.append('$sd', sdq) + return DbUtils.seaweed.find(query).toList() + } catch (Exception e) { + e.printStackTrace() + } + return 0 + } + + + static def findThirdPartyPrice(def city, def region, def name, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) + + def dbcursor = thirdpartColl.find(q).sort(new BasicDBObject("s_date", -1)) + def map = [:] + for(def db : dbcursor){ + def key = db.city+"##"+db.region+"##"+db.name + def dataMap = map.get(key) + if(!dataMap){ + dataMap = [:] + map.put(key, dataMap) + } + + if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){ + dataMap.put("anjuke_esf_price", db.anjuke_esf_price) + } + if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){ + dataMap.put("anjuke_esf_price", db.get("58tc_esf_price")) + } + + if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){ + dataMap.put("lianjia_esf_price", db.lianjia_esf_price) + } + if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){ + dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price) + } + + if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){ + dataMap.put("beike_esf_price", db.beike_esf_price) + } + } + return map + } + + + static def findGXDPrice(def city, region, name){ + + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("district_name", name) + q.put("source", "GXD") + def dbcursor = confirm_price_date.find(q).sort(new BasicDBObject("c_date", -1)).limit(1) + def price = null + for(def d : dbcursor){ + price = d.price + } + return price + } + + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy index 34fa4eefe..6ed286549 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy @@ -3,6 +3,7 @@ package com.fangjia.script.gj import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.Bytes @@ -25,7 +26,18 @@ class AddAvgpriceBatch { static main(args) { def task = new AddAvgpriceBatch(); // task.run1(); - task.test() +// task.test() + task.sendPriceCommand() + } + + def sendPriceCommand(){ + ExcelUtils.multiThreadingModify("D:\\数据\\美术宝\\美术宝测试数据(价格).xlsx", { row -> + if(!row['P'] && row['T'] && ["别墅","普通住宅","商住两用"].contains(row['T'])){ + //发重算均价命令 + println row['K']+" "+row['L']+" "+row['M'] + addPrice(row['K'],row['L'],row['M'], row['T'], DateUtils.parseFromTightByPattern('2023-10-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-10-25','yyyy-MM-dd')) + } + }) } def test(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 9da0ae032..441134cee 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -207,7 +207,14 @@ class FangjiaClientUtils { try { address = address.replace("{","") // def url = "http://192.168.0.19:39080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8") - def url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&city="+URLEncoder.encode(city,"UTF-8")+"&category=${property}&useMap=${useMap}" + def url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" + + if(city){ + url = url + "&city="+URLEncoder.encode(city,"UTF-8") + } + if(property){ + url = url + "&category=${property}" + } // def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8") content = new HttpRequest().newGet(url, null, "utf-8") jsonObj = JSONObject.fromObject(content) @@ -218,4 +225,19 @@ class FangjiaClientUtils { return jsonObj; } + public static def newStandard(address){ + def jsonObj = null + def content = null + try { + address = address.replace("{","") + def url = "http://192.168.0.19:31080/FangJiaNlpService/format?address="+URLEncoder.encode(address,"UTF-8") + content = new HttpRequest().newGet(url, null, "utf-8") + jsonObj = JSONObject.fromObject(content) + } catch (Exception e) { + println address +" "+content + e.printStackTrace(); + } + return jsonObj; + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meishubao/MsbPocTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meishubao/MsbPocTest.groovy new file mode 100644 index 000000000..cf326c49a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meishubao/MsbPocTest.groovy @@ -0,0 +1,227 @@ +package com.fangjia.script.gj.meishubao + +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient + +class MsbPocTest { + + static main(args) { + def task = new MsbPocTest(); + task.run(); + } + def run(){ +// def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(小区、城市均价).xlsx" + def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(匹配V2).xlsx"//美术宝测试数据(匹配).xlsx +// standard(inputfile) +// match(inputfile) +// matchOld(inputfile) +// addPrice("D:\\数据\\美术宝\\美术宝测试数据(价格).xlsx") +// test("D:\\数据\\美术宝\\test2.xlsx") + + cityPrice() + } + + def cityPrice(){ + def datas = ExcelUtils.read("D:\\数据\\临时\\城市房产价格.xlsx",0,1) + def map = [:] + for(def d : datas){ + map.put(d['A'], d) + } + + def ccbdatas = ExcelUtils.read("D:\\数据\\临时\\建行城市均价.xlsx",0,1) + def ccbmap = [:] + for(def d : ccbdatas){ + ccbmap.put(d['A'], d['B']) + } + + ExcelUtils.modify("D:\\数据\\临时\\房价网城市均价.xlsx", { row -> + if(!row['C']){ + def d = map.get(row['A']) + if(!d){ + d = map.get(row['A'].replace('州','')) + } + if(d){ + row['C'] = d[1] + row['D'] = d[2] + } + + + def d1 = ccbmap.get(row['A']) + if(!d1){ + d1 = ccbmap.get(row['A'].replace('州','')) + } + if(d1){ + row['E'] = d1 + } + } + + }) + + } + + + + def matchOld(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['S'] && row['D']){ + def city = row['C'] + def region = row['D'] + def name = row['F'] + def address = row['H'] + + println "[match] city:"+city+" region:"+region+" address:"+address + def jsonObject = OpenApiClient.addressMatch(city, address, null, false, region, name) + if(jsonObject != null && jsonObject.precise!=-1) { + row['S'] = jsonObject.get("id") + row['T'] = jsonObject.get("city") + row['U'] = jsonObject.get("district") + row['V'] = jsonObject.get("name") + row['W'] = jsonObject.get("propertyType") + row['X'] = jsonObject.get("credit") + } + } + }) + } + + + + def test(file){ + ExcelUtils.multiThreadingModify(file, { row -> + def jsonObject = FangjiaClientUtils.newMatch(null, row['A'], null, true) + println row['A'] + if(jsonObject != null && jsonObject.credit>0) { + row['B'] = jsonObject.get("id") + row['C'] = jsonObject.get("city") + row['D'] = jsonObject.get("district") + row['E'] = jsonObject.get("name") + def type = jsonObject.get("propertyType") + if(type && type!='null'){ + row['F'] = type + } + row['G'] = jsonObject.get("credit") + } + }) + } + + def match(file){ + ExcelUtils.multiThreadingModify(file, { row -> + if(!row['J'] && row['F'] && row['P']=="没匹配" ){ + def city = row['C'] + def region = row['D'] + def address = row['G'] //row['G']//row['F'] + + println "[match] city:"+city+" region:"+region+" address:"+address + def jsonObject = FangjiaClientUtils.newMatch(city, address, null, false) + if(jsonObject != null && jsonObject.credit>0) { + row['J'] = jsonObject.get("id") + row['K'] = jsonObject.get("city") + row['L'] = jsonObject.get("district") + row['M'] = jsonObject.get("name") + def type = jsonObject.get("propertyType") + if(type && type!='null'){ + row['N'] = type + } + row['O'] = jsonObject.get("credit") + } + } + }) + } + + def addPrice(path){ + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + ExcelUtils.multiThreadingModify(path, { row -> + if(row['I'] && !row['P']){ + row['I'] = row['I'].toString().trim() + def d = CommonDatautils.findDistrictById(row['I']) + if(d){ + row['K'] = d.city + row['L'] = d.region + row['M'] = d.name + row['T'] = d.estate_type2 + row['U'] = d.cat + row['V'] = d.status + + def cityMap = CommonDatautils.findThirdPartyPrice(d.city, d.region, d.name, 202303,202310) + def priceMap = cityMap.get(d.city+"##"+d.region+"##"+d.name) + if(priceMap){ + if(priceMap.containsKey("lianjia_esf_price")){ + row['N'] = priceMap.get("lianjia_esf_price") + } + + if(priceMap.containsKey("beike_esf_price")){ + row['N'] = priceMap.get("beike_esf_price") + } + + if(priceMap.containsKey("anjuke_esf_price")){ + row['O'] = priceMap.get("anjuke_esf_price") + } + } + + + def dbo = CommonDatautils.queryFirstDistrictPrice(d.city, d.region, d.name) + if(dbo){ + row['P'] = dbo.avg_price + } + + def cityId = cityCodeDataMap.get(d.city) + def map = CcbCommonUtils.findAvgPrice(row['I'], cityId) + if(map){ + row['Q'] = map.avg_price + } + def tradePrice = CommonDatautils.findTrades(d, '2023-03-01','2023-10-01') + if(tradePrice){ + row['R'] = tradePrice.toInteger() + } + + def gxdPrice = CommonDatautils.findGXDPrice(d.city, d.region, d.name) + d['S'] = gxdPrice + } + + + } + }) + } + + def standard(inputfile){ + ExcelUtils.multiThreadingModify(inputfile, { row -> + if(row['G']){ + + row['G'] = row['F'].replace('·','').replace("小区","").replace("花园","").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期").replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区") +// println "[match] city:"+city+" region:"+region+" address:"+address +// def jsonObject = FangjiaClientUtils.newStandard(row['G']) +// if(jsonObject != null && jsonObject.precise!=-1) { +// row['Q'] = jsonObject.get("name") +// row['R'] = jsonObject.get("id") +// } + } + }) + } + + + def addThirdPrice(path){ + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + + ExcelUtils.multiThreadingModify(path, { row -> + def cityMap = CommonDatautils.findThirdPartyPrice(row['D'], row['E'], row['F']) + def priceMap = cityMap.get(row['D']+"##"+row['E']+"##"+row['F']) + if(priceMap){ + if(priceMap.containsKey("lianjia_esf_price")){ + row['L'] = priceMap.get("lianjia_esf_price") + } + + if(priceMap.containsKey("beike_esf_price")){ + row['M'] = priceMap.get("beike_esf_price") + } + + if(priceMap.containsKey("anjuke_esf_price")){ + row['N'] = priceMap.get("anjuke_esf_price") + } + } + + }) + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/PoiTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/PoiTest.groovy new file mode 100644 index 000000000..6bc14bab2 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/PoiTest.groovy @@ -0,0 +1,80 @@ +package com.fangjia.script.gj.test + +import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.GeoUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils + + +//def json = GaoDeApiUtils.geo("上海市", "上海市浦东新区永泰路1138弄11号") +//def locationStr = null +//if(json && json.count>=1){ +// def m = json.geocodes[0] +// if(m){ +// locationStr = m.location +// } +//} + +def location = ["lng":121.526016,"lat":31.143047] + + +//def rows = [] +//rows << ["城市","区域","名称","地址","距离","均价"] +//if(location && location.lat){ +// +// def dbos = CommonDatautils.getDistrictArround("上海", 121.526016, 31.143047) +// for(def d : dbos){ +// +// def dbo = CommonDatautils.findDistrictPrice(d.city, d.region, d.name); +// def avg_price = null +// if(dbo){ +// avg_price = dbo.avg_price +// } +// +// def distance = GeoUtils.calculateDistance(location.lng, location.lat, d.lng/10000000000.0d, d.lat/10000000000.0) +// +// rows << [d.city, d.region, d.name, d.address, distance.intValue(), avg_price, d.status] +// } +// +//// def map = CommonDatautils.findMaxLatLngMinLatLng(location.lat, location.lng, 1000); +//// def ret = CommonDatautils.findInRangeDist("上海", "district", (map.maxLat * 10000000000d).toLong(), (map.minLat * 10000000000d).toLong(), (map.maxLng * 10000000000d).toLong(), (map.minLng * 10000000000d).toLong()); +//// +//// if(ret){ +//// for(def row : ret){ +//// if(row.lat2 && row.lng2){ +//// def lat2 = row.lat2/10000000000.0; +//// def lng2 = row.lng2/10000000000.0; +//// region = row.region; +//// def distance = GeoHelper.GetDistance(location.lat, location.lng, lat2, lng2) +//// if(distance <= 1000){ +//// def dbo = CommonDatautils.findDistrictPrice(row.city, row.region, row.name); +//// def avg_price = null +//// if(dbo){ +//// avg_price = dbo.avg_price +//// } +//// rows << [row.city, row.region, row.name, row.address, distance.intValue(), avg_price] +//// } +//// } +//// } +//// } +// +// ExcelUtils.write("D:\\数据\\临时\\上海市浦东新区永泰路1138弄11号.xlsx", rows, "一公里楼盘") +//} + + +ExcelUtils.multiThreadingModify("D:\\数据\\临时\\地图查询数据导出-添加经纬度-兴趣点.xlsx",{d-> + def loc = d['AB'] + if(loc){ + def arrs = loc.split(",") + def distance = GeoUtils.calculateDistance(location.lng, location.lat, arrs[0].toString().toDouble(), arrs[1].toString().toDouble()) + d['AD'] = distance + } +}) + + + + +//def distance = GeoUtils.calculateDistance(121.526016, 31.143047, 111.202375, 35.592230) +//println distance diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy index 4dad6f8ca..da19a39b6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy @@ -1,12 +1,14 @@ package com.fangjia.script.gj.xiamen import com.fangjia.core.util.GeoUtils +import com.fangjia.script.BaseBatch import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.DateUtil import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import com.rabbitmq.client.AMQP import org.bson.types.ObjectId import java.util.concurrent.CopyOnWriteArrayList @@ -16,13 +18,15 @@ import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong import com.fangjia.script.common.DataHandleUtils -class SeaweedDataSynch { +class SeaweedDataSynch extends BaseBatch{ static def targetUrl = "http://192.168.0.12:9200/seaweed/" public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL) + DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL) + public static def provinceMap = CommonDatautils.findAllCityProvince() def lastMonthDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); @@ -30,14 +34,27 @@ class SeaweedDataSynch { static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com") + + public static final DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020) + + + static main(args) { - def task = new SeaweedDataSynch() -// def addr = ["a","b",'c',"d","e","f"] -// println addr[0..4] - task.run() -// println new ObjectId("622c433afb459ab08fdab460").getDate() + runBatch(new SeaweedDataSynch(), args); } + @Override + public void process(Object args) { + run() + println "全国district, shop, office同步完成" + //民政部编码上传 + //codeRun() + //println "民政部省,市,区同步完成" + //println new ObjectId("622c433afb459ab08fdab460").getDate() + } + + + def run(){ priceMap = findAllDistrictSellPrice() def citys = CommonDatautils.findAllCity() @@ -52,7 +69,7 @@ class SeaweedDataSynch { } def updating(city, cat){ - def dists = CommonDatautils.findAllDistricts(city, cat, ["city","region","name","address","block","mapping_alias","addr_alias","cat","estate_type2","status","township","lng","lat","build_type","household_count","committee","estate_charge","car_site_count","property_life","house_property","build_area","all_area","build_structure","developer","estate_company","cubage_percent","virescence_percent","loudong_count","schools","main_type","main_type_area","create_year"]) + def dists = CommonDatautils.findAllDistrictsFromMg(city, cat, ["city","region","name","address","block","mapping_alias","addr_alias","cat","estate_type2","status","township","lng","lat","build_type","household_count","committee","estate_charge","car_site_count","property_life","house_property","build_area","all_area","build_structure","developer","estate_company","cubage_percent","virescence_percent","loudong_count","schools","main_type","main_type_area","create_year"]) def rets = new CopyOnWriteArrayList() def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(dists.size()) @@ -208,4 +225,27 @@ class SeaweedDataSynch { } return priceMap } + + def codeRun(){ + + def q = new BasicDBObject() + def dbcursor = mca_administrative_division_coll.find(q) + def rets = [] + for(def it : dbcursor){ + def udata = new BasicDBObject() + udata.put("_id", it._id) + udata.put("province", it.province) + udata.put("city", it.city) + udata.put("district", it.district) + udata.put("mca_province_code", it.province_code) + udata.put("mca_city_code", it.city_code) + udata.put("mca_district_code", it.district_code) + udata.put("level", it.level) + udata.put("c_date", it.c_date) + rets << udata +// administrative_division_100_CONN.save(udata) +// administrative_division_100_CONN.update(new BasicDBObject('_id', it._id), new BasicDBObject('$set', udata), true, false) + } + administrative_division_100_CONN.insert(rets) + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy index abf289d1f..85d25ddaf 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy @@ -414,7 +414,8 @@ public class ExcelUtils{ def data = ExcelUtils.read(fileName, srcSheetIndex, 0) def body = data.subList(1, data.size()) - def nThread = Runtime.getRuntime().availableProcessors() * 2 + def nThread = Runtime.getRuntime().availableProcessors() * 5 + println nThread ExecutorService executorService = Executors.newFixedThreadPool(nThread) final AtomicInteger counter = new AtomicInteger(0) final double total = body.size()