From 2f8d203a7079af66e7868c42aed5e094b242d354 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 22 Oct 2024 15:14:16 +0800 Subject: [PATCH 01/13] =?UTF-8?q?=E5=88=A0=E9=99=A4=E9=87=8D=E5=A4=8D?= =?UTF-8?q?=E5=B0=8F=E5=8C=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/gj/district/RemoveDistrict.groovy | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/RemoveDistrict.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/RemoveDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/RemoveDistrict.groovy new file mode 100644 index 000000000..2f49dc227 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/RemoveDistrict.groovy @@ -0,0 +1,56 @@ +package com.fangjia.script.gj.district + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DbUtils +import com.mongodb.BasicDBObject + +/** + * 临时移除 相同名称即在正常状态下有,在删除状态下也有的小区 + */ +class RemoveDistrict { + + public static void main(String[] args) { + def task = new RemoveDistrict() + task.run() + } + + def run(){ + def citys = CommonUtils.findAllCity() + def cat = "shop" + def keys = [] + citys.each {c-> + def districts = CommonDatautils.findAllDistricts(c, cat, ["city", "region", "name", "status"]) + districts.each {it-> + def key = it.city +"##"+it.region+"##"+it.name + if(!keys.contains(key)){ + keys << key + }else{ + println it.toString()+" 删除" + DbUtils.seaweed.remove(new BasicDBObject("_id",it._id)) + } +// def list = CommonDatautils.findAllStatusDistrict(it.city, it.region, it.name, cat, ["city", "region", "name", "status"]) +// if(list && list.size()>1){ +// println it.toString() + " "+list.size() +// for(def d : list){ +//// if(d.status.toString()=="0"){ +//// println d.toString()+" 保留" +//// }else if(d.status.toString()=="1"){ +//// println d.toString()+" 删除" +//// DbUtils.seaweed.remove(new BasicDBObject("_id",d._id)) +//// } +// +// if(d.status.toString()=="1"){ +// println d.toString()+" 保留" +// }else if(d.status.toString()!="1"){ +// println d.toString()+" 删除" +// DbUtils.seaweed.remove(new BasicDBObject("_id",d._id)) +// } +// } +// +// println "................................................................................................" +// } + } + } + } +} From 352c454b2c14d98d92ab27c58ede818af9c32663 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 22 Oct 2024 15:14:41 +0800 Subject: [PATCH 02/13] =?UTF-8?q?=E5=B0=8F=E5=8C=BA=E5=9B=BE=E7=89=87?= =?UTF-8?q?=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/images/AddDistImagesBatch.groovy | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/images/AddDistImagesBatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/images/AddDistImagesBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/images/AddDistImagesBatch.groovy new file mode 100644 index 000000000..9ba1fe9ca --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/images/AddDistImagesBatch.groovy @@ -0,0 +1,146 @@ +package com.fangjia.script.gj.images + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.bson.types.ObjectId +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 更新小区图片 + */ +@BatchOwner(Watcher.GaoJuan) +class AddDistImagesBatch extends BaseBatch{ + final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018) + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + + public static final DBCollection seaweed_fs_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_images", "mongo.db.fangjia.com") + + static main(args) { + runBatch(new AddDistImagesBatch(), args); + } + + @Override + public void process(Object args) { +// def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd") + def mdate = DateUtils.getFirstDayOfMonth(new Date(), -1) + def citys = CommonUtils.findAllCity() + citys.each {city-> + def communitys = findAllEsData2(city, mdate) + println "抓取城市: "+city +" 第三方小区有图片数:"+communitys.size() + //第三方匹配小区 + matchCommunitys(communitys) + } + } + + + + def findAllEsData2(city, mdate){ + QueryBuilder date_range = QueryBuilders.rangeQuery("m_date").from(mdate.getTime()).includeLower(true) + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("source", "安居客小程序")) + .must(date_range).must(date_range) + + def rets = [] + def datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "community", queryBuilder); + if(datas) { + for (int k = 0; k < datas.size(); k++) { + def d = datas[k]; + if(d.house_type_images_2 || d.outdoor_images) { + def data = new BasicDBObject() + data.put("city", d.city) + data.put("region", d.region) + data.put("name", d.name) + data.put("id", d.id) + data.put("out_images", d.outdoor_images) + data.put("house_type_images", d.house_type_images_2) + rets << data + } + } + } + return rets; + } + + def findAllEsData(city, mdate){ + def names = new HashSet() + def rets = [] + ElasticUtils.executeSqlQuery("select * from community where source='安居客小程序' and city='${city}' and m_date>='${mdate}' order by m_date", { d-> + if(d.house_type_images_2 || d.outdoor_images){ + def s = d.city +"#"+d.region+"#"+d.name+"#"+d.property_type + if(!names.contains(s)){ + names << s + def data = new BasicDBObject() + data.put("city", d.city) + data.put("region", d.region) + data.put("name", d.name) + data.put("out_images", d.outdoor_images) + data.put("house_type_images", d.house_type_images_2) + rets << data + } + } + }) + return rets; + } + + + def matchCommunitys(communitys){ + def addCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(communitys.size()) + def rets = new CopyOnWriteArrayList() + def total = communitys.size() + for(def i=0;i Date: Tue, 22 Oct 2024 15:15:34 +0800 Subject: [PATCH 03/13] =?UTF-8?q?=E6=B7=B1=E5=9C=B3=E6=88=90=E4=BA=A4?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/ccbevaluation/AddSzDealData.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy index 974df17ee..0bf6ec4b9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy @@ -9,7 +9,7 @@ import com.mongodb.DBCollection /** * 深圳公众号成交数据录入 */ -def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年7月深圳二手房成交.xlsx", 0, 1) +def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年9月深圳二手房成交.xlsx", 0, 1) def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) From 2701a125d80f3473f37a94eb08e0f42f963f222a Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 22 Oct 2024 15:16:56 +0800 Subject: [PATCH 04/13] =?UTF-8?q?=E6=96=B0=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index f9d5867e0..d8e78df14 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -40,7 +40,7 @@ class CcbCommonUtils { public static final DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com") - +// public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "deal_price_all", "mongo.db.fangjia.com") public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017) From 40a338302d50e4bd4ceca3951c7e8152e40f2748 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 22 Oct 2024 15:17:50 +0800 Subject: [PATCH 05/13] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E8=A1=A5=E5=85=85?= =?UTF-8?q?=E5=9F=8E=E5=B8=82=EF=BC=8C=E5=8C=BA=E5=9F=9F=EF=BC=8C=E5=B0=8F?= =?UTF-8?q?=E5=8C=BA=E5=9D=87=E4=BB=B7=E5=92=8C=E7=A7=9F=E9=87=91=E7=9A=84?= =?UTF-8?q?=E5=91=BD=E4=BB=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/data_handle/AddCityRegionPrice.groovy | 115 +++++++++++++++++- 1 file changed, 112 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy index cc42938d7..954e0402b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy @@ -6,21 +6,33 @@ import com.fangjia.script.averager.InsertMonitorSquareRentCalcCommandBatch import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.MongoClient +import com.rabbitmq.client.AMQP class AddCityRegionPrice { public final static DBCollection COMMAND_COLL = new MongoClient("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('calc_command') + + public static final def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017"; + public static final DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName2("fangjia", "sqmetre_rent", mongoDbUrl) + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", mongoDbUrl) + static main(args) { def task = new AddCityRegionPrice() // task.addRegionPrice() -// task.addCityprice() + task.addCityprice() // task.addDistrictprice() // task.addCityRentPrice() - task.addDistrictRentprice() +// task.addRegionRentPrice() +// task.addDistrictRentprice() +// task.updateRegionRentData() +// task.updateDealPriceAreaType() +// task.checkCityImport() } def addRegionPrice(){ @@ -54,7 +66,7 @@ class AddCityRegionPrice { def addCityprice(){ InsertMonitorCalcCommandBatch insertBatch = new InsertMonitorCalcCommandBatch() - def citys = CommonDatautils.findAllCity().reverse() + def citys = ["乐东"]// CommonDatautils.findAllCity().reverse() def add = 0 for(def c : citys){ def startObj = CommonDatautils.queryCityOrRegionPrice(c, null, 1) @@ -125,6 +137,35 @@ class AddCityRegionPrice { } } + def addRegionRentPrice(){ + InsertMonitorSquareRentCalcCommandBatch inserBatch = new InsertMonitorSquareRentCalcCommandBatch(); + def citys = CommonDatautils.findAllCity() + for(def c : citys){ + + def regions = CommonDatautils.getCityRegions(c) + for(def r : regions){ +// if(r!="南川"){ +// continue +// } + def startObj = CommonDatautils.queryFirstOrLastRentPrice(c, r, null, 1) + def endObj = CommonDatautils.queryFirstOrLastRentPrice(c, r, null, -1) + if(startObj && endObj){ + def months = DateUtils.getBetweenMonth(DateUtil.formatByPattern(startObj.weekend, "yyyyMM"),"202409") + months.each {m-> +// println m + def lastDay = DateUtils.truncateDay(DateUtils.getLastDayOfMonth(DateUtil.parseDate(m+"01", "yyyyMMdd"))) + def obj = CommonDatautils.findSqmetreRent(c, r, null, lastDay) + if(!obj){ + inserBatch.saveCityCalcCommand(c, r, null, lastDay, "fangjia-priceanalysis-0") + println "[add] "+c+" "+r+" "+DateUtil.formatByPattern(lastDay, "yyyy-MM-dd") + } + } + } + } + + } + } + def addDistrictRentprice(){ InsertMonitorSquareRentCalcCommandBatch inserBatch = new InsertMonitorSquareRentCalcCommandBatch(); def citys = CommonDatautils.findAllCity() @@ -158,4 +199,72 @@ class AddCityRegionPrice { } } + + def updateRegionRentData(){ + def q = new BasicDBObject("cat", "region") + def keys = new BasicDBObject() + keys.put("markup", 1) + def dbcursor = SQMETRE_RENT_COLL.find(q, keys).batchSize(5000) + def index = 1 + for(def d : dbcursor){ + if(!(d.markup instanceof Double)){ + def udata = new BasicDBObject() + double markup = 1.0 + markup = d.markup * markup + udata.put("markup", markup) + SQMETRE_RENT_COLL.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + println index + index++ + } + } + } + + + def updateDealPriceAreaType(){ + def index = 1 + def citys = ["凉山州","福州","南京","佛山","沈阳","苏州"] + citys.each {c-> + def q = new BasicDBObject() + q.put("city", c) +// q.put("price_flag", "国信达成交-推导") + def dbcursor = DEAL_PRICE_COLL.find(q) + for(def d : dbcursor){ +// if(!(d.area instanceof Double)){ +// def udata = new BasicDBObject() +// double area = 1.0 +// area = d.area * area +// udata.put("area", area) +// DEAL_PRICE_COLL.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) +// println c+" "+index +// index++ +// } + if(!(d.total_price instanceof Integer)){ + def udata = new BasicDBObject() + udata.put("total_price", d.total_price.toString().toDouble().toInteger()) + DEAL_PRICE_COLL.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + println c+" "+index + index++ + } + } + } + } + + def checkCityImport(){ + def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"] + for(def c : citys){ + def startObj = CommonDatautils.findStatIndexBySort("城市", c, null, 1) + def endObj = CommonDatautils.findStatIndexBySort("城市", c,null, -1) + if(startObj && endObj){ + def months = DateUtils.getBetweenMonth(startObj.yearmonth.toString(),"202409") + months.each {m-> + def obj = CommonDatautils.findStatIndexByMonth("城市", c,null, m.toInteger()) + if(!obj){ + println "[add] "+c+" "+m + } + } + }else{ + println c +" 一个也没有" + } + } + } } From d6a12be1afe39ac35a652d1abb7318814847ff78 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 22 Oct 2024 15:20:52 +0800 Subject: [PATCH 06/13] =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/gj/poc/AddPriceTest.groovy | 36 ++++++++++++++++--- .../thirdPriceCrawl/DataStandardBatch.groovy | 2 +- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 4 +++ 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy index 14b6a5c26..eed8d9fda 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy @@ -25,6 +25,10 @@ class AddPriceTest { def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse", "postgres", "postgres", "org.postgresql.Driver") + //235为房价网内部计算均价的库 + def pgsqlInstance235 = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified", + "postgres", "Goojia123456", "org.postgresql.Driver") + static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") public static final DBCollection confirm_price_date = MongoUtils.getCollectionByName("fangjia", "confirm_price_data", "mongo.db.fangjia.com") @@ -33,7 +37,7 @@ class AddPriceTest { task.run(); } def run(){ - def filePath = "D:\\数据\\建行\\房产估值\\20240716-20240815 _业务反馈价格.xlsx" + def filePath = "D:\\数据\\建行\\房产估值\\建行业务反馈badcase.xlsx" // addGXDPrice(filePath) addCcbPrice(filePath) // addThirdPrice(filePath) @@ -173,17 +177,22 @@ class AddPriceTest { def cityId = cityCodeDataMap.get(d['V']) def map = findAvgPrice(d['U'], cityId) if(map){ - d['F'] = map.avg_price -// d['P'] = map.calc_type + d['H'] = map.avg_price + d['I'] = map.calc_type // d['AM'] = map.credit_value } + +// def newPriceMap = findNewFjwAvgPrice(d['U'], cityId) +// if(newPriceMap){ +// d['Q'] = newPriceMap.avg_price +// } } }) } def findAvgPrice(def commid, def cityid){ - def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202407 order by gujiabds_clc_yrmo_val desc limit 1" + def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202409 order by gujiabds_clc_yrmo_val desc limit 1" def map = [:] pgsqlInstance.eachRow(sql, { r -> map.avg_price = r.gujia_bds_clc_avprc @@ -202,6 +211,25 @@ class AddPriceTest { } + def findNewFjwAvgPrice(def commid, def cityid){ + def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from price_stat.avm_community_base_price where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202409 order by gujiabds_clc_yrmo_val desc limit 1" + def map = [:] + pgsqlInstance235.eachRow(sql, { r -> + map.avg_price = r.gujia_bds_clc_avprc + map.type = r.gujia_bds_clmtd + map.month = r.gujiabds_clc_yrmo_val + map.credit_value = r.gujia_bds_cfdc_dgr + if(r.gujia_bds_clmtd == 1){ + map.calc_type = "人工估值" + }else if(r.gujia_bds_clmtd == 2){ + map.calc_type = "样本计算" + }else if(r.gujia_bds_clmtd == 3){ + map.calc_type = "神经网络" + } + }) + return map + } + def addThirdPrice(filepath){ ExcelUtils.multiThreadingModify(filepath, { d -> if (!d['X']) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index 1d63a790f..b6e1b0ced 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -62,7 +62,7 @@ class DataStandardBatch extends BaseBatch{ private static def districtMap = [:]; private static def limit = 5000; - def static final SHORT_DATE = Integer.parseInt(com.fangjia.script.util.DateUtils.addMonth(com.fangjia.script.util.DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); + def static final SHORT_DATE = Integer.parseInt(com.fangjia.script.util.DateUtils.addMonth(com.fangjia.script.util.DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6, "无名称":7] def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋","幸福里","中国房价行情","常州住房"] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 668fcbb74..5f07adb27 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -76,6 +76,10 @@ class SynchDistrictPrice extends BaseBatch{ data.put("build_type",d.build_type) data.put("c_date", new Date()) data.put("m_date", new Date()) +// if(d.location){ +// data.put("lng", d.location.x) +// data.put("lat", d.location.y) +// } rets << data } }) From c859830a101ff263d5f0c0503ed8525c5d65a713 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 22 Oct 2024 15:21:47 +0800 Subject: [PATCH 07/13] =?UTF-8?q?=E7=AC=AC=E4=B8=89=E6=96=B9=E5=9D=87?= =?UTF-8?q?=E4=BB=B7=E8=A1=A8=E4=BF=AE=E6=94=B9=E5=8C=BA=E5=9F=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/data_handle/RegionUpdateBatch.groovy | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionUpdateBatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionUpdateBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionUpdateBatch.groovy new file mode 100644 index 000000000..6fce88fa6 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionUpdateBatch.groovy @@ -0,0 +1,65 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +//更新第三方均价表中小区区域 +class RegionUpdateBatch { + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50) + + static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + + static main(args) { + def task = new RegionUpdateBatch() + task.run() + + } + def run(){ + def citys = CommonDatautils.findAllCity() + for(def city : citys){ + def districtMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["city","region", "name"]) + def keys = new BasicDBObject() + keys.put("city", 1) + keys.put("region", 1) + keys.put("name", 1) + keys.put("seaweed_id", 1) + def statistics = AVERAGE_STATISTICS.find(new BasicDBObject("city", city), keys) + def total = statistics.size() + final CountDownLatch latch = new CountDownLatch(total) + def totalCount = new AtomicLong() + for(def i=0;i Date: Tue, 22 Oct 2024 15:22:23 +0800 Subject: [PATCH 08/13] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=B0=8F=E5=8C=BA?= =?UTF-8?q?=E5=9B=B4=E6=A0=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../data_handle/AddDistrictShapeBatch.groovy | 286 ++++++++++++++++++ 1 file changed, 286 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy new file mode 100644 index 000000000..5093ecb3a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy @@ -0,0 +1,286 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.rabbitmq.client.impl.AMQImpl + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 添加高德地图,百度地图围栏 + */ +@BatchOwner(Watcher.GaoJuan) +class AddDistrictShapeBatch extends BaseBatch{ + + def fullCityMap = CommonDatautils.queryFullNameCity() + def mcaDistMap = CommonDatautils.queryDistNameCityByMapping() + + DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020) + public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com") + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) + + static main(args) { + runBatch(new AddDistrictShapeBatch(), args) + } + + @Override + public void process(Object args) { + def citys = CommonDatautils.findAllCity() + def index = 1 + for(def c : citys){ + println c +" "+index + index ++ + def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2"]) +// baiduMatch(c, districts) + gaodeMatch(c, districts) + + } + } + + def baiduMatch(city,districts){ + def data = fullCityMap.get(city) + def fullCityName = null + if(data){ + fullCityName = data.city + } + final CountDownLatch latch = new CountDownLatch(districts.size()) + + if(fullCityName){ + def rets = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + for(def i=0;i1) { +// resultMap.type = "MultiPolygon" +// }else{ + resultMap.type = "polygon" +// } + o.put("shape", resultMap) + + def q = new BasicDBObject("comm_id", d._id.toString()) + seaweed_fence_coll.update(q, new BasicDBObject('$set', o),true,false) +// seaweed_fence_coll.save(o) + println "[save] 围栏 ( "+d.city +" "+d.region +" "+d.name+" )" + return fenceMap.data_id + } + return null + } + + def findBaiduDatas(city, district, name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("district", district) + q.put("name", name) + q.put('std_tag', '房地产;住宅区') + q.put("guoke_geo", new BasicDBObject('$ne', null)) + def dbcursor = BAIDU_POI_COLL.find(q) + def map = [:] + println dbcursor.count() + for(def d : dbcursor){ + def arrs = d.gcj02_guoke_geo.split("\\|") + + def coordinatesStr = arrs[2].toString().substring(2, arrs[2].toString().size()-1 ) + // 将字符串分割成列表 + def coordinatePairs = coordinatesStr.split(',') + def coordinatesList = [] + for (int i = 0; i < coordinatePairs.size(); i += 2) { + // 将字符串坐标转换为浮点数 + double longitude = coordinatePairs[i] as double + double latitude = coordinatePairs[i + 1] as double + + // 将坐标点作为一个元组添加到集合中 + coordinatesList << [longitude, latitude] + } + if(coordinatePairs){ + map.data_id = d._id + map.coordinatePairs = coordinatesList + } + } + return map; + } + + + def gaodeMatch(city,districts){ + def data = fullCityMap.get(city) + def fullCityName = null + if(data){ + fullCityName = data.city + } + final CountDownLatch latch = new CountDownLatch(districts.size()) + + if(fullCityName){ + def rets = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + for(def i=0;i + def newName = d.name.replace("-","").replace("·","") + if(newName == name && d.type.contains("住宅") && d.shape){ + obj = d + } + }) + return obj; + } + def findSeaweedFenceExists(commid){ + def q = new BasicDBObject() + q.put("comm_id", commid) + return seaweed_fence_coll.findOne(q) + } + + +} From a1a06d0d8f8a1efe538bda82a592f4dc91c0ebca Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 22 Oct 2024 15:22:57 +0800 Subject: [PATCH 09/13] =?UTF-8?q?=E8=87=AA=E5=8A=A8=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E5=B0=8F=E5=8C=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../data_handle/AutoAddDistrictBatch.groovy | 234 ++++++++++++++++++ 1 file changed, 234 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy new file mode 100644 index 000000000..f27ee361f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy @@ -0,0 +1,234 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.BaseBatch +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 小区自动入库(新增小区,每月运行一次) + */ +class AutoAddDistrictBatch extends BaseBatch{ + + final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018) + public static DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") + DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020) + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + + def fullCityMap = CommonDatautils.queryFullNameCity() + def mcaDistMap = CommonDatautils.queryDistNameCityByMapping() + + static main(args) { + runBatch(new AutoAddDistrictBatch(), args) + } + + @Override + void process(Object args) { + //查找第三方匹配没有匹配上的小区 + def citys = CommonDatautils.findAllCity() + def list = [] + list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "高德地图"] + for(def c : citys){ + def noMatchDatas = findNoMatchResult(c) + println c +" 新旧算法未匹配的数据量:"+noMatchDatas.size() + def rets = checkDatas(noMatchDatas.keySet()) + list.addAll(rets) + } + + def fileName = "/home/goojia/gj/add_dist_info.xlsx" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\数据建设\\名录集\\第三方待入库数据.xlsx" + } + ExcelUtils.write(fileName, list, "待入库数据") + } + + def checkDatas(keysets){ + def noMatchCount = keysets.size() + final CountDownLatch latch = new CountDownLatch(noMatchCount) + + def rets = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + for(def i=0;i + if(d.name == name && d.type.contains("住宅") && d.shape){ + obj = d + } + }) + return obj; + } + + def findTrades(city, region, name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("district_name", name) + return DEAL_PRICE_COLL.count(q) + } + + def findNoMatchResult(def c){ + def q = new BasicDBObject() + q.put("match_id", new BasicDBObject('$exists',false)) + q.put("old_match_id", new BasicDBObject('$exists',false)) + q.put("city", c) + def dbcursor = thirdSeaweedMappingColl.find(q).batchSize(5000) + def rMap = [:] + for(def d : dbcursor){ + if(d.estate_type){ + if(d.estate_type.contains("自建房")){ + continue + } + if(d.estate_type.contains("购物中心")){ + continue + } + if(d.estate_type.contains("住宅底商") || d.estate_type.contains("酒店") || d.estate_type.contains("商铺")){ + continue + } + if(d.estate_type.contains("办公") || d.estate_type.contains("写字楼")){ + continue + } + }else{ + continue + } + def key = d.city + "##"+d.district+"##"+d.name + def rets = rMap.get(key) + if(!rMap.containsKey(key)){ + rets = [] + rMap.put(key, rets) + } + rets << d + } + return rMap + } + + def createData(city, region, name, address, lng, lat){ + def udata = new BasicDBObject() + udata.put("city", city) + udata.put("region", region) + name = name.replaceAll('[·\\(\\)()]|·', '').replace("建设中", "") + udata.put("name", name) + + def addr = AddressUtil.formatAddress(city, address) + if(addr){ + udata.put("address", addr) + } + + udata.put("status", "0") + udata.put("name_cat", "0") + udata.put("cat", "district") + udata.put("username", "高娟") + + def estateType2 = "普通住宅" + def type = "" + if(type){ + if(type.contains("别墅")){ + estateType2 = "别墅" + }else if(type.contains("商住")){ + estateType2 = "商住" + } + } + udata.put("estateType2", estateType2) + if(lng){ + Long gd_lng = lng*10000000000 + udata.put("lng", gd_lng) + } + if(lat){ + Long gd_lat = lat * 10000000000 + udata.put("lat", gd_lat) + } + FangjiaClientUtils.saveOrUpdateDistrict(udata) + } + + def findBaiduDatas(city, district, name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("district", district) + q.put("name", name) + q.put('std_tag', '房地产;住宅区') + def count = BAIDU_POI_COLL.count(q) + return count; + } +} From ea56f1fd1c9dc530cc956470c9b3cadbe7a6b025 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 22 Oct 2024 15:23:50 +0800 Subject: [PATCH 10/13] =?UTF-8?q?=E4=B9=A1=E9=95=87=E8=A1=97=E9=81=93?= =?UTF-8?q?=E4=B8=AD=E5=BF=83=E7=82=B9=E6=A3=80=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/data_handle/TownCenterBatch.groovy | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownCenterBatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownCenterBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownCenterBatch.groovy new file mode 100644 index 000000000..e1bfa8969 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownCenterBatch.groovy @@ -0,0 +1,79 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +/** + * 乡镇街道 判断 中心点是否在围栏或一个围栏是否包含多个中心点 + */ +@BatchOwner(Watcher.GaoJuan) +class TownCenterBatch extends BaseBatch{ + + DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com") + + static main(args) { + runBatch(new TownCenterBatch(), args) + } + + @Override + void process(Object args) { + def shapes = findHasShapeTown() + def centers = findAllCenters() + checkCenter(shapes, centers) + + } + + def checkCenter(shapes, centers){ + def map = [:] + for(def town : shapes){ + def points = InsidePolygon.createPoint4(town.shape.coordinates) + //1.先判断当前镇的中心点是否在围栏中 + if(town.location){ + def flag = InsidePolygon.isInArea(town.location.lng, town.location.lat, points) + if(!flag){ + println town.city + " "+town.district+" "+town.street+" 不在本身围栏中" + } + } + //2.判断是否有其它点在围栏中 + for(def center : centers){ + if(center._id.toString() != town._id.toString()){ + def flag = InsidePolygon.isInArea(center.location.lng, center.location.lat, points) + if(flag){ + def list = map.get(town.city + "#"+town.district+"#"+town.street) + if(!list){ + list = [] + map.put(town.city + "#"+town.district+"#"+town.street) + } + list << center.city + "#"+center.district+"#"+center.street + println center.city + " "+center.district+" "+center.street+" 在 "+town.city + " "+town.district+" "+town.street+" 围栏中" + } + } + } + } + return map + } + + def findHasShapeTown(){ + def q = new BasicDBObject() + q.put("type", "乡镇街道") + q.put("shape", new BasicDBObject('$exists', true)) + return ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(2000).toList() + } + + def findAllCenters(){ + def q = new BasicDBObject() + q.put("type", "乡镇街道") + q.put("location", new BasicDBObject('$exists', true)) + def keys = new BasicDBObject() + keys.put("location", 1) + keys.put("city", 1) + keys.put("district", 1) + keys.put("street", 1) + return ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).toList() + } +} From 1d8e9fa91388974034ff1c9631dfe3a8feb26c3c Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 22 Oct 2024 15:24:55 +0800 Subject: [PATCH 11/13] =?UTF-8?q?=E5=B0=8F=E5=8C=BA=E5=AD=97=E6=AE=B5?= =?UTF-8?q?=E8=87=AA=E5=A1=AB=E5=85=85=E5=A2=9E=E5=8A=A0=E7=BB=8F=E7=BA=AC?= =?UTF-8?q?=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/district/DistrictFieldAddBatch.groovy | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy index 76fa40fb2..3b3aed61a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy @@ -47,7 +47,7 @@ class DistrictFieldAddBatch extends BaseBatch{ //"alias":"alias", //ownership_type:权属类型 为商品房、经适房、动迁房等 //house_property 产权性质:国有产权 "house_property":"网站没有", - def fieldsMap = ["estate_charge":"property_fee","block":"block","build_date":"build_year","estate_company":"property_company","developer":"developer", + def fieldsMap = ["lng":"location","estate_charge":"property_fee","block":"block","build_date":"build_year","estate_company":"property_company","developer":"developer", "virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area","household_count":"households", "car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address","build_structure":"building_structure", "build_type":"build_type", "land_property":"ownership_type","live_date":"live_date", @@ -115,7 +115,7 @@ class DistrictFieldAddBatch extends BaseBatch{ println "【通过匹配填充字段】网站:"+seaweedmapSource +" Es:"+esRets.size()+" 融合小区占比:"+DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/totalCount)+"%" def precentMap = fieldUpdate(totalCount, updateRets) println c +" "+precentMap.toString() - //updateDB(c, updateRets) + updateDB(c, updateRets) println "------------------------------------------------------------------------------" def pre1 = 0 def pre2 = 0 @@ -212,6 +212,11 @@ class DistrictFieldAddBatch extends BaseBatch{ def rets = [] rets << ["第三方城市","第三方区域","第三方名称","第三方地址","第三方户数","匹配区域","匹配名称","匹配可信度","方式"] for(def matchDist : matchRets){ + +// if(!["智富中心","龙山新寓","天怡瀚园","远盛凤林府","京口路小区","梦溪路小区","浩创优尚","溪山丽苑","旭园"].contains(matchDist.name)){ +// continue +// } + def dist = CommonDatautils.findDistrictFieldById(matchDist.match_id, fields) if(!dist){ println c+" "+matchDist.match_id+" 不存在" @@ -306,6 +311,15 @@ class DistrictFieldAddBatch extends BaseBatch{ if(newVal){ newValues.put(seaweedfield, newVal) } + if(seaweedfield == "lng"){ + def locationMap = FiledStrand.processLngAndLat(esVal) + if(locationMap){ + newValues.put("lat", locationMap.lat) + newValues.put("lng", locationMap.lng) + newValues.put("lat2", locationMap.lat2) + newValues.put("lng2", locationMap.lng2) + } + } } } } From e1fbc1ee6a5daea2fea1b11f89a0e7a01b467ee1 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 22 Oct 2024 15:32:18 +0800 Subject: [PATCH 12/13] update --- .../ccbevaluation/AddThirdPriceSample.groovy | 82 +++++++++++-- .../fangjia/script/ccbevaluation/Test.groovy | 100 ++++++++-------- .../script/ccbevaluation/TradeStatLog.groovy | 21 +++- .../ccbevaluation/data/AddressCut.groovy | 97 +++++++++------ .../script/commData/CommonDatautils.groovy | 75 ++++++++++-- .../com/fangjia/script/gj/FiledStrand.groovy | 24 ++++ .../com/fangjia/script/gj/TradeMatch.groovy | 96 ++++++++++++--- .../script/gj/data_handle/DevProcess.groovy | 4 +- .../gj/data_handle/DevUpdateTestBatch.groovy | 10 +- .../gj/data_handle/GXDdistrictMatch.groovy | 28 ++++- .../data_handle/ThirdPriceMappingBatch.groovy | 9 +- .../gj/data_handle/TownMiddleProcess.groovy | 112 +++++++++++++----- .../script/trade/ImportIndexStats.groovy | 85 +++++++++---- 13 files changed, 556 insertions(+), 187 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPriceSample.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPriceSample.groovy index 2684f90c2..a36d9d715 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPriceSample.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPriceSample.groovy @@ -13,19 +13,24 @@ class AddThirdPriceSample { static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + public static final DBCollection NO_PRICE_DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "deal_price_all", "mongo.db.fangjia.com") + + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") + static main(args) { def task = new AddThirdPriceSample() - task.run() + task.run() //添加第三方 fccs->保存的是国信达的参考价格 csjj->保存的是国信达的成交数据 +// task.addTrade() //添加成交 } def run(){ - for(def i=1;i<19;i++){ - ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\级联待优化小区集合_7月.xlsx",{d-> - if(d['A'] && d['C']){ + for(def i=0;i<19;i++){ + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\国信达参考价作为第三方入库.xlsx",{d-> + if(d['A'] && d['B']){ def dist = CommonDatautils.findDistrictById(d['A'].toString().trim()) if(dist){ - def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202407) - def udata = new BasicDBObject("csjj_esf_price", d['C'].toString().toDouble().toInteger()) - + def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202408) +// def udata = new BasicDBObject("csjj_esf_price", d['C'].toString().toDouble().toInteger()) + def udata = new BasicDBObject("fccs_esf_price", d['B'].toString().toDouble().toInteger()) AVERAGE_STATISTICS.update(q,new BasicDBObject('$set', udata),false, false) }else{ println d['A'] +" 没查到" @@ -37,4 +42,67 @@ class AddThirdPriceSample { } } + + def addTrade(){ + def citys = ["武汉","南京","杭州","长沙","郑州","合肥","宁波","南通","嘉兴","珠海","泉州","绍兴","湖州","镇江"] + def rets = findTrades(citys) +// DEAL_PRICE_COLL.insert(rets) + } + + def findTrades(citys){ + def q = new BasicDBObject() + q.put("city", new BasicDBObject('$in', citys)) + q.put("flag", 1) + def dbcursor = NO_PRICE_DEAL_PRICE_COLL.find(q) + def rets = [] + def index = 0 + for(def db : dbcursor){ + index++ + def udata = new BasicDBObject() + + udata.put("_id",db._id) + udata.put("url",db.url) + udata.put("area",db.area) + udata.put("avg_price",db.avg_price.toString().toDouble().toInteger()) + udata.put("build_structure",db.build_structure) + udata.put("build_type",db.build_type) + udata.put("build_year",db.build_year) + udata.put("city",db.city) + udata.put("direction", db.direction) + udata.put("district_name",db.district_name) + udata.put("elevator",db.elevator) + udata.put("fitment",db.fitment) + udata.put("floor_info",db.floor_info) + udata.put("hall",db.hall.toString().toInteger()) + udata.put("height",db.height.toString().toInteger()) + udata.put("housing_age",db.housing_age) + udata.put("lift",db.lift.toString().toInteger()) + udata.put("lift_household",db.lift_household.toString().toInteger()) + udata.put("property_ownership",db.property_ownership) + udata.put("region",db.region) + udata.put("room", db.room.toString().toInteger()) + udata.put("room_structure",db.room_structure) + udata.put("source",db.source) + udata.put("total_price",db.total_price.toString().toInteger()) + udata.put("trade_date",db.trade_date) + udata.put("trade_property",db.trade_property) + udata.put("house_using",db.house_using) + udata.put("heating_type",db.heating_type) + udata.put("community_id",db.community_id) + udata.put("create_date",db.create_time) + udata.put("m_date", new Date()) + udata.put("address",db.address) + udata.put("match_city",db.match_city) + udata.put("match_region",db.match_region) + udata.put("match_name",db.match_name) + udata.put("match_id",db.match_id) + udata.put("price_flag", "国信达成交-推导") + + DEAL_PRICE_COLL.save(udata) + println index +// rets << udata + } + return rets + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 0d36523e6..fed8e9757 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -631,36 +631,36 @@ def findAllData(sqlInstance236, citycode){ //},0) -ExcelUtils.modify("D:\\数据\\建行\\房产估值\\添加别名.xlsx",{d-> - if(d['A'] && d['B']){ - def dbo = CommonDatautils.findDistrictById(d['A']) - if(dbo){ - def alias = null - if(!dbo.alias){ - alias = [] - }else{ - alias = dbo.alias - } - if(!alias.contains(d['B'])){ - alias << d['B'] - } - if(alias){ - def udata = new BasicDBObject() - udata.put("alias", alias) - udata.put("m_date",new Date()) - - def q = new BasicDBObject() - q.put("_id", dbo._id) - DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) - }else{ - println d['A']+" "+"别名已存在" - } - - }else{ - println dbo._id.toString() +" 没找到" - } - } -}) +//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\添加别名.xlsx",{d-> +// if(d['A'] && d['B']){ +// def dbo = CommonDatautils.findDistrictById(d['A']) +// if(dbo){ +// def alias = null +// if(!dbo.alias){ +// alias = [] +// }else{ +// alias = dbo.alias +// } +// if(!alias.contains(d['B'])){ +// alias << d['B'] +// } +// if(alias){ +// def udata = new BasicDBObject() +// udata.put("alias", alias) +// udata.put("m_date",new Date()) +// +// def q = new BasicDBObject() +// q.put("_id", dbo._id) +// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +// }else{ +// println d['A']+" "+"别名已存在" +// } +// +// }else{ +// println dbo._id.toString() +" 没找到" +// } +// } +//}) //def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"] //for(def c : citys){ @@ -1331,34 +1331,34 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ -//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240806--国信达价格录入.xlsx" +def fileName = "D:\\数据\\建行\\房产估值\\6个城市待优化_9月xlsx.xlsx" // -//ExcelUtils.multiThreadingModify(fileName, {d-> -// if(d['J']){ -//// def region = RegionData.formatRegion(d['A'], d['J']) -//// if(region){ -//// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district") -//// if(dist){ -//// d['B'] = dist.region -//// d['C'] = dist.name -//// d['L'] = dist.status-dist.name_cat -//// } -//// } -//// if(d['J'] instanceof String){ -// def dist = CommonDatautils.findDistrictById(d['J']) +ExcelUtils.multiThreadingModify(fileName, {d-> + if(d['G']){ +// def region = RegionData.formatRegion(d['A'], d['J']) +// if(region){ +// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district") // if(dist){ +// d['B'] = dist.region +// d['C'] = dist.name +// d['L'] = dist.status-dist.name_cat +// } +// } + if(d['G'] instanceof String){ + def dist = CommonDatautils.findDistrictById(d['G']) + if(dist){ // d['A'] = dist.city // d['B'] = dist.region // d['C'] = dist.name // d['K'] = dist.estate_type2 // d['L'] = dist.status // d['M'] = dist.name_cat -// -//// d['H'] = dist.name -// } -//// } -// } -//},0) + + d['H'] = dist.name + } + } + } +},0) //def file = "D:\\数据\\建行\\房产估值\\指导价\\5城价格翻新6-21.xlsx" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index 9fc014cca..c2caf475f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -47,7 +47,8 @@ class TradeStatLog { // task.run()//统计最近3个月抓取的成交数据量 // task.exportTrade()//导出未入库成交 - task.exportTrade2()//导出未入库成交 +// task.exportTrade2()//导出未入库成交 + task.updateDealPriceRegion()//修改成交区域数据 // task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) // task.averageStat() @@ -597,7 +598,7 @@ class TradeStatLog { // def endDate = "2024-07-01" def ids = findAllDealIds(); println ids.size() - ExcelUtils.multiThreadingModify("D:\\数据\\其它\\2024年6月成交数据.xlsx",{d-> + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\建行成交\\廊坊.xlsx",{d-> if(ids.contains(d['A'])){ d['V'] = "供" } @@ -674,7 +675,7 @@ class TradeStatLog { } def findAllDealIds(){ - def sql = "select gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where p9_start_date='2024-07-28'" + def sql = "select gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where urbn_nm='廊坊'" def ids = [] pssqlInstance.eachRow(sql,{r-> if(!ids.contains(r.gujia_bds_mdl_id)){ @@ -801,6 +802,20 @@ class TradeStatLog { },0) } + def updateDealPriceRegion(){ + ExcelUtils.modify("D:\\数据\\建行\\建行成交\\成交待修改区域数据.xlsx",{d-> + if(d['A'] && d['G']){ + def region = RegionData.formatRegion(d['D'], d['G']) + if(region){ + def udata = new BasicDBObject() + udata.put("region", region) + DEAL_PRICE_COLL.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata)) + } + + } + },0) + } + def findTradeById(id){ def q = new BasicDBObject() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy index af276a9c9..10a3cc392 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy @@ -24,6 +24,13 @@ class AddressCut extends BaseBatch{ final DBCollection thirdPartyPriceColl = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018) + final DBCollection thirdPartyPriceBkColl = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price_history_bk_new", "192.168.0.235", 27018) + + + final DBCollection dataProcessColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process", "192.168.0.235", 27018) + + final DBCollection dataProcessBkColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process_history_bk", "192.168.0.235", 27018) + public static EsDataSource dataSource = new EsDataSource() private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000" static { @@ -39,50 +46,66 @@ class AddressCut extends BaseBatch{ @Override public void process(Object args) { -<<<<<<< HEAD // sameAddr() - seaweedStreetUpdate() + def dates = [202408] + for(def date : dates){ +// synch(date) + synchDataProcess(date) + println date + "同步完成! " + } + } - def seaweedStreetUpdate(){ - ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区地址拆分\\已核实-路2.xlsx",{d-> - if(d['A']){ - def dbo = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district") - if(dbo){ - def udate = new BasicDBObject() - def street = null - def address = "" - if(d['E']){ - street = d['E'] - udate.put("street", street) - address = street - } - def streetNum = null - if(d['F']){ - streetNum = d['F'] - udate.put("street_number", streetNum) - address = address + streetNum - } - if(dbo.address!=address){ - udate.put("address", address) - } - if(udate){ - udate.put("m_date", new Date()) - DBUtils.seaweed.update(new BasicDBObject('_id', dbo.get("_id")), new BasicDBObject('$set', udate)) - } - } + def synch(date){ + def q = new BasicDBObject() + q.put("s_date", date) + def dbcursor = thirdPartyPriceColl.find(q).batchSize(5000) + def index = 0 + def rets = [] + def num = 0 + for(def d : dbcursor){ + num++ + index++ + rets << d + if(index%10000==0){ + thirdPartyPriceBkColl.insert(rets) + println(num) + rets.clear() } - - }) -======= -// sameAddr() //相同地址小区 -// roadMatch() - addAnjukeRoad() - + } + if(rets.size()>0){ + thirdPartyPriceBkColl.insert(rets) + rets.clear() + } } + def synchDataProcess(date){ + def q = new BasicDBObject() + q.put("s_date", date) + def dbcursor = dataProcessColl.find(q).batchSize(5000) + def index = 0 + def rets = [] + def num = 0 + for(def d : dbcursor){ + num++ + index++ + rets << d + if(index%10000==0){ + dataProcessBkColl.insert(rets) + println(num) + rets.clear() + } + } + if(rets.size()>0){ + dataProcessBkColl.insert(rets) + rets.clear() + } + } + + + def addAnjukeRoad(){ ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区地址拆分\\小区地址切分(结果)_1.xlsx", { d -> def roads = findThirdPartyPrice(d['A'], d['B'], d['C'], "链家") @@ -130,8 +153,6 @@ class AddressCut extends BaseBatch{ } }) - ->>>>>>> c1a5ab8c6aa973a91e6be5547ec0925db3fa500f } def sameAddr() { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 94668a4df..5e2937768 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -122,6 +122,8 @@ class CommonDatautils { public static final DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com") public static final DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com") + public static final DBCollection STATS_INDEX_COLL = MongoUtils.getCollectionByName("fangjia_stat", "stats_index", "mongo.db.fangjia.com") + public static DSCollection getHouseDataStoreConnection(){ if(house_dataStoreConnection != null) { return house_dataStoreConnection @@ -820,6 +822,7 @@ class CommonDatautils { DBObject q = new BasicDBObject(); q.put("status",0) +// q.put("status",new BasicDBObject('$ne', 0)) q.put("cat", cat); q.put("city", city) @@ -829,7 +832,7 @@ class CommonDatautils { keys.put(field, 1); } } - def dbsour = SEAWEED_COLL.find(q, keys).batchSize(2000) + def dbsour = SEAWEED_COLL.find(q, keys).batchSize(5000) def rets = [] for(def d : dbsour){ rets << d @@ -959,13 +962,7 @@ class CommonDatautils { keys.put(field, 1); } } - def cursor = getDataStoreConnection().find(q, keys).batchSize(2000) - if(cursor){ - for(def obj : cursor){ - return obj; - } - } - return null; + return SEAWEED_COLL.findOne(q, keys) } @@ -1132,6 +1129,10 @@ class CommonDatautils { q.put("city", city); q.put("cat", cat); q.put("name", name); + if(region){ + q.put("region", region); + } + q.put("status",new BasicDBObject('$ne', 0)) DBObject keys = new BasicDBObject(); if(fields){ @@ -1680,7 +1681,7 @@ class CommonDatautils { DBObject queryCondition = new BasicDBObject() queryCondition.put("city", city); queryCondition.put("category","region"); - queryCondition.put("show",1); +// queryCondition.put("show",1); def dbCursor = REGION_BLOCK_COLL.find(queryCondition) def regions = []; for(DBObject obj : dbCursor){ @@ -2385,6 +2386,9 @@ class CommonDatautils { q.put("city", city); if(region){ q.put("region", region); + q.put("cat", "区域") + }else{ + q.put("cat", "城市") } q.put('weekend', weekend); def dbCursor = REGION_STAT_COLL.find(q).sort(new BasicDBObject().append("weekend", -1)).limit(1); @@ -3735,9 +3739,9 @@ class CommonDatautils { if(!region){ if(city){ q.put("cat", "city") - }else{ - q.put("cat", "region") } + }else{ + q.put("cat", "region") } } @@ -4095,6 +4099,21 @@ class CommonDatautils { return retMap; } + + static def findHouseCount(def city, def region, def name){ + + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) + .must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("district_name", name)).must(QueryBuilders.termQuery("cat", "sell")) + + + def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"] + def index = new Random().nextInt(5); + def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder); + return houses.size(); + } + + /** * 是否是别墅 * @return @@ -5060,4 +5079,38 @@ class CommonDatautils { } + static def findStatIndexBySort(type, city, region, sort){ + def q = new BasicDBObject() + q.put("cat", "district") + q.put("type", type) + if(region){ + q.put("district", region) + } + if(city){ + q.put("city", city) + } + + def cur = STATS_INDEX_COLL.find(q).sort(new BasicDBObject("yearmonth", sort)).limit(1) + cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def obj = 0; + while(cur.hasNext()) { + obj = cur.next() + } + return obj; + } + + static def findStatIndexByMonth(type, city, region, month){ + def q = new BasicDBObject() + q.put("cat", "district") + q.put("type", type) + if(region){ + q.put("district", region) + } + if(city){ + q.put("city", city) + } + q.put("yearmonth", month) + return STATS_INDEX_COLL.findOne(q) + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy index 1b11ee16a..a3c119bec 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy @@ -2,8 +2,10 @@ package com.fangjia.script.gj import com.fangjia.core.BlockData import com.fangjia.core.RegionData +import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.core.util.DateUtil import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.linxsh.util.AddressUtil import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils @@ -531,6 +533,28 @@ class FiledStrand { return null } + + static def processLngAndLat(val) { + if (val) { + def map = [:] + def lng = DataHandleUtils.getDoubleRound(6, val.x) + def lat = DataHandleUtils.getDoubleRound(6, val.y) + Long gd_lat = lat * 10000000000L + Long gd_lng = lng * 10000000000L + map.lat = gd_lat + map.lng = gd_lng + + def baidu = BaiDuApiUtils.geoconv(lng, lat, 3) + Long bd_lng = baidu.x * 10000000000L + Long bd_lat = baidu.y * 10000000000L + map.lat2 = bd_lat + map.lng2 = bd_lng + return map + + } + return null + } + static def isRoad(def s){ def pattern = Pattern.compile("[路|街|大道|巷]"); def matcher = pattern.matcher(s) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TradeMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TradeMatch.groovy index 082e2f170..a496ed93f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TradeMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TradeMatch.groovy @@ -2,33 +2,97 @@ package com.fangjia.script.gj import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.DataUtil import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection -ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\成交未匹配.xlsx",{d-> - if(d['A'] && !d['E']){ -// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'].toString(), "district") -// def dist = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'].toString(), "district") - println d['A'] +" "+d['C'] -// def total = findDatas(d['A'], d['B'], d['C']) -// d['D'] = total - def c = getAmapLocation(d['A']+"市", d['B']+"区", d['C'].replace("?","").replace("·","").replace("?","").replace("|","").replace("·","")) - d['E'] = c - } -}) +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +//ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\成交未匹配.xlsx",{d-> +// if(d['A'] && !d['E']){ +//// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'].toString(), "district") +//// def dist = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'].toString(), "district") +// println d['A'] +" "+d['C'] +//// def total = findDatas(d['A'], d['B'], d['C']) +//// d['D'] = total +// def c = getAmapLocation(d['A']+"市", d['B']+"区", d['C'].replace("?","").replace("·","").replace("?","").replace("|","").replace("·","")) +// d['E'] = c +// } +//}) + +def URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; +DBCollection seaweed_coll = MongoUtils.getCollectionByName2("fangjia", "seaweed", URL) +def q = new BasicDBObject() +q.put("city", "上海市") +q.put("category", "district") +q.put("status", 0) +def ret = [] +def list = seaweed_coll.find(q).batchSize(5000).toList() +println "总计:"+list.size() +//for(def l : list){ +// def obj = getAmapLocation(l.city, l.region, l.name) +// if(obj){ +// ret << [l.city, l.region, l.name, obj.cityname, obj.adname, obj.name,obj.type] +// } +//} + +ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) +def addCount = new AtomicLong() +final CountDownLatch latch = new CountDownLatch(list.size()) +def total = list.size() +for(def i=0;i - if(d.name.contains(name) && d.type.contains("住宅")){ - total ++ + if(d.name.contains(name) && d.type.contains("住宅") && d.shape){ + obj = d } }) - return total; + return obj; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index 56915936b..f722db94c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -46,7 +46,7 @@ class DevProcess { def task = new DevProcess(); // task.run() // task.parseCityAndDist() - task.devSplitRoad() +// task.devSplitRoad() // task.devNameSplitRoad() // task.addFlag() // task.matchDev() @@ -63,7 +63,7 @@ class DevProcess { // task.createDevMapping() // task.deleteAddr() // task.deleteAddr2() -// task.synchDevTo100() + task.synchDevTo100() // task.addNewDev2() // task.addAlias() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy index d56afba8a..cf1b49203 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy @@ -20,6 +20,10 @@ class DevUpdateTestBatch { def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; DBCollection development_zone_43_CONN = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com") + + + DBCollection development_zone_100_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL) + DBCollection DEV_MAPPING_100_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL) @@ -30,8 +34,8 @@ class DevUpdateTestBatch { static main(args) { def task = new DevUpdateTestBatch() def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx -// task.addDev(path) - task.addDevAddress(path) + task.addDev(path) +// task.addDevAddress(path) // // task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是") @@ -42,7 +46,7 @@ class DevUpdateTestBatch { def addDev(path){ ExcelUtils.modify(path,{ d-> if(!d['I'] && d['A']){ - def obj = development_zone_100_CONN.findOne(new BasicDBObject("name", d['A'])) + def obj = development_zone_43_CONN.findOne(new BasicDBObject("name", d['A'])) if(!obj){ def o = new BasicDBObject() o.put("province", d['B']) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy index 35c1fa9db..ee54817e6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy @@ -28,7 +28,7 @@ class GXDdistrictMatch { static main(args) { def task = new GXDdistrictMatch(); // task.run() - task.addOldMatch() + task.addOldMatch2() // task.runDb() // task.updateDistStatus()//恢复name_cat不正常数据 @@ -68,6 +68,32 @@ class GXDdistrictMatch { } + def addOldMatch2(){ + //老匹配 + def fileName = "D:\\数据\\建行\\房产估值\\建行业务反馈badcase.xlsx"//11城重新匹配.xlsx + ExcelUtils.multiThreadingModify(fileName, { d -> + println d['C'] + if(!d['AJ']){ + def jsonObject = OpenApiClient.addressMatch(d['C'], d['D'], "district", false,null, d['D']) + if (jsonObject != null && !jsonObject.containsKey("result")) { + d['AJ'] = jsonObject.get("id") + d['AK'] = jsonObject.get("city") + d['AL'] = jsonObject.get("district") + d['AM'] = jsonObject.get("name") + d['AN'] = jsonObject.get("address") + + def dbo = CommonDatautils.findDistrictById(jsonObject.get("id")) + if(dbo){ + d['AO'] = dbo.status + d['AP'] = dbo.name_cat + d['AQ'] = dbo.estate_type2 + } + } + } + }); + + } + def runDb(){ def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx" ExcelUtils.modify(fileName, { d -> diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy index 0fbf352f7..41d940274 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy @@ -35,7 +35,7 @@ class ThirdPriceMappingBatch extends BaseBatch{ final DBCollection dataProcessColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process", "192.168.0.235", 27018) - def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); + def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); static main(args) { runBatch(new ThirdPriceMappingBatch(), args); @@ -44,9 +44,9 @@ class ThirdPriceMappingBatch extends BaseBatch{ @Override public void process(Object args) { // //将新增数据同步过来 -// addNemCommunity() + addNemCommunity() // //接口进行匹配 - match() +// match() // addOldMatchResult() } @@ -151,8 +151,9 @@ class ThirdPriceMappingBatch extends BaseBatch{ sd.each {k,v-> udata.put(k,v) } + udata.removeField("_id") udata.put("m_date", new Date()) - thirdSeaweedMappingColl.update(query, new BasicDBObject('$set', udata)) + thirdSeaweedMappingColl.update(query, new BasicDBObject('$set', udata),false, false) }else{ sd.put("m_date", new Date()) thirdSeaweedMappingColl.save(sd) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy index 0f61101ea..9d5283a6b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy @@ -1,6 +1,7 @@ package com.fangjia.script.gj.data_handle import com.fangjia.core.RegionData +import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.core.util.GeocodeUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils @@ -900,41 +901,92 @@ class TownMiddleProcess { } def updateTownPolineBatch(){ + def i = 0 ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\乡镇街道围栏开发区数据.xlsx",{ row-> if(row['R'] && row['R']=="新增围栏"){ - def q = new BasicDBObject("city", row['B']).append("region", row['C']).append("town",row['D']) - println row['B']+" "+ row['D']+" "+TOWNPOLYLINE_COLL.count(q) - def d = TOWNPOLYLINE_COLL.findOne(q) - if(d){ - def resultMap = [:] - resultMap.type = "polygon" - def arrs = d.polyline.split("\\|") - def rets = [] - for(def s : arrs){ - def points = s.split(";") - def pointList = [] - for(def pointStr : points){ - def pointArr = pointStr.split(",") - def lon = DataHandleUtils.getDoubleRound(6, pointArr[0].toString().toDouble()) - def lat = DataHandleUtils.getDoubleRound(6, pointArr[1].toString().toDouble()) - pointList << [lon, lat] + i++ + if(i>=134){ + def q = new BasicDBObject("city", row['B']).append("region", row['C']).append("town",row['D']) + println row['B']+" "+ row['D']+" "+TOWNPOLYLINE_COLL.count(q) + def d = TOWNPOLYLINE_COLL.findOne(q) + if(d){ + def resultMap = [:] + def bd_resultMap = [:] + + resultMap.type = "polygon" + bd_resultMap.type = "polygon" + + def arrs = d.polyline.split("\\|") + def rets = [] + def bd_rets = [] + + println row['N'] +" 共有:"+arrs.size()+" 个围栏" + def index = 1 + for(def s : arrs){ + def points = s.split(";") + def pointList = [] + def bd_pointList = [] + + for(def pointStr : points){ + def pointArr = pointStr.split(",") + def lon = DataHandleUtils.getDoubleRound(6, pointArr[0].toString().toDouble()) + def lat = DataHandleUtils.getDoubleRound(6, pointArr[1].toString().toDouble()) + pointList << [lon, lat] + + // 高德转百度 + def baidu = BaiDuApiUtils.geoconv(lon, lat, 3) + def bd_lng = baidu.x + def bd_lat = baidu.y + bd_pointList << [bd_lng, bd_lat] + } + + if(index<=1){ + resultMap.coordinates = pointList + bd_resultMap.coordinates = bd_pointList + + def udata = new BasicDBObject() + udata.put("shape", resultMap) + udata.put("operator", "高娟") + udata.put("m_date", new Date()) + udata.put("bd_shape",bd_resultMap) + + def updateQuery = new BasicDBObject("name",row['N']) + DEVELOPMENT_ZONE_43_COLL.update(updateQuery, new BasicDBObject('$set',udata)) + }else{ + def dbo = DEVELOPMENT_ZONE_43_COLL.findOne(new BasicDBObject("name",row['N'])) + def obj = new BasicDBObject() + dbo.each {k,v-> + if(k!='_id'){ + obj.put(k,v) + } + } + + resultMap.coordinates = pointList + bd_resultMap.coordinates = bd_pointList + obj.put("shape", resultMap) + obj.put("operator", "高娟") + obj.put("m_date", new Date()) + obj.put("bd_shape",bd_resultMap) + obj.put("create_time", new Date()) + obj.put("block", dbo.name+"-地块"+index) + DEVELOPMENT_ZONE_43_COLL.save(obj) + } + +// rets << pointList +// bd_rets << bd_pointList + index++ } - rets << pointList + + +// if(rets.size()>1) { +// resultMap.type = "MultiPolygon" +// bd_resultMap.type = "MultiPolygon" +// } +// println d.city+" "+d.region+" "+d.town+" "+rets.size() + row['S'] = "已完成" } - resultMap.coordinates = rets - if(rets.size()>1) { - resultMap.type = "MultiPolygon" - } - println d.city+" "+d.region+" "+d.town+" "+rets.size() - - def udata = new BasicDBObject() - udata.put("shape", resultMap) - - - def updateQuery = new BasicDBObject("name",row['N']) - DEVELOPMENT_ZONE_43_COLL.update(updateQuery, new BasicDBObject('$set',udata)) - row['S'] = "已完成" } + } }) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy index c51b3f825..b5de80cae 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy @@ -53,7 +53,7 @@ class ImportIndexStats extends BaseBatch{ - public static final def dingjiYear = 2020; + public static def dingjiYear = 2020; private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" private static final EsDataSource dataSource = new EsDataSource() @@ -76,22 +76,23 @@ class ImportIndexStats extends BaseBatch{ // synch() // synchTjjIndex()//同步统计局抓取过来的指数 -// Calendar calendar=Calendar.getInstance(); -// def month= calendar.get(Calendar.MONTH); -// def year= calendar.get(Calendar.YEAR); -// def date = createDateString(year, month, 0, ""); -// calculatePriceIndex(year, month, date); + Calendar calendar=Calendar.getInstance(); + def month= calendar.get(Calendar.MONTH); + def year= calendar.get(Calendar.YEAR); + def date = createDateString(year, month, 0, ""); + calculatePriceIndex(year, month, date); // addLostIndex() - - addHistoryIndex() +// addHistoryIndex() } def addLostIndex(){ def citys = CommonDatautils.findAllCity(); + def index = 0 for(def city : citys){ + index++ if(cityList.contains(city)){ - println city +" 数据来源于统计局" + println "[${index}] "+ city +" 数据来源于统计局" continue; } def start = findStartOrEndIndex(city, 1) @@ -117,21 +118,48 @@ class ImportIndexStats extends BaseBatch{ if(nextObj && nextObj.chain){ indexs << nextObj.chain } - udata.put("chain",DataHandleUtils.getDoubleRound(1, indexs.sum()/indexs.size())) - udata.put("remark","推导") + if(indexs){ + udata.put("chain",DataHandleUtils.getDoubleRound(1, indexs.sum()/indexs.size())) + udata.put("remark","推导") + } + } + if(!dbo.yeartoyear){ + def indexs = [] + if(lastMonthObj && lastMonthObj.yeartoyear){ + indexs << lastMonthObj.yeartoyear + } + if(nextObj && nextObj.yeartoyear){ + indexs << nextObj.yeartoyear + } + if(indexs){ + udata.put("yeartoyear",DataHandleUtils.getDoubleRound(1, indexs.sum()/indexs.size())) + udata.put("remark","推导") + } + } + if(!dbo.fixedbase){ + def indexs = [] + if(lastMonthObj && lastMonthObj.fixedbase){ + indexs << lastMonthObj.fixedbase + } + if(nextObj && nextObj.fixedbase){ + indexs << nextObj.fixedbase + } + if(indexs){ + udata.put("fixedbase",DataHandleUtils.getDoubleRound(1, indexs.sum()/indexs.size())) + udata.put("remark","推导") + } + + } + if(udata){ + println city +" 补充" STATS_INDEX_COLL.update(new BasicDBObject("_id", dbo._id), new BasicDBObject('$set', udata),false, true); } -// if(!dbo.yeartoyear){ -// -// } -// if(!dbo.fixedbase){ -// -// } - } } } + + println "[${index}] "+ city +" 已处理完" } } @@ -154,7 +182,7 @@ class ImportIndexStats extends BaseBatch{ } def addHistoryIndex(){ - def s = "202101" + def s = "202001" def startDate = null; def stop = false; def runDates = [] @@ -171,7 +199,7 @@ class ImportIndexStats extends BaseBatch{ runDates << map s = DateUtils.addMonth(date, 1, "yyyyMM") - if(year==2024 && month==9){ + if(year==2024 && month==8){ stop = true; } } @@ -182,6 +210,7 @@ class ImportIndexStats extends BaseBatch{ //计算统计局没有发布的城市的价格指数 def calculatePriceIndex(def year, def month, def date){ +// def citys = ["五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]// def citys = CommonDatautils.findAllCity(); // def date = createDateString(year, month, 0, ""); def haveIndexCitys = findPriceIndexCitys(date.toInteger()); @@ -193,7 +222,7 @@ class ImportIndexStats extends BaseBatch{ def newCitys = [] for(def i=0;i=201601){ + dingjiYear = 2015 + }else if(date.toInteger()>=202101){ + dingjiYear = 2020 + }else if(date.toInteger()>=202601){ + dingjiYear = 2025 + }else if(date.toInteger()>=203101){ + dingjiYear = 2030 + } fixedThreadPool.execute(new Runnable() { void run() { try{ @@ -220,7 +258,7 @@ class ImportIndexStats extends BaseBatch{ } }); } - latch.await(20, TimeUnit.SECONDS); + latch.await(1, TimeUnit.MINUTES); } def calculating(city, year, month, date){ @@ -647,6 +685,7 @@ class ImportIndexStats extends BaseBatch{ DBObject q = new BasicDBObject(); q.put("yearmonth", date); q.put("cat", "district") + q.put("type", "城市") DBObject keys = new BasicDBObject(); keys.put("city", 1) @@ -666,6 +705,7 @@ class ImportIndexStats extends BaseBatch{ q.put("yearmonth", date); q.put("city", city); q.put("cat", "district"); + q.put("type", "城市") DBObject keys = new BasicDBObject(); keys.put("city", 1) @@ -684,6 +724,7 @@ class ImportIndexStats extends BaseBatch{ dbo.put("source",result.source); dbo.put("remark",result.remark); dbo.put("cat",result.cat); + dbo.put("type",result.type); dbo.put("c_date",new Date()); if(result.markup){ dbo.put("chain", result.markup.toDouble()); From bda9cca523525788cff9c7934024e253e4e6eec2 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 22 Oct 2024 18:24:49 +0800 Subject: [PATCH 13/13] update --- .../DealAndThirdPriceDataProcess.groovy | 89 +++++++++++++------ 1 file changed, 63 insertions(+), 26 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index 023cc157d..d2c8a1f84 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -5,9 +5,14 @@ import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject import groovy.sql.Sql import org.bson.types.ObjectId +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + /** * 建行成交数据表优化 */ @@ -24,6 +29,8 @@ class DealPriceDataProcess { def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified", "postgres", "postgres", "org.postgresql.Driver") + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + static main(args) { def task = new DealPriceDataProcess() // task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据 @@ -394,43 +401,73 @@ class DealPriceDataProcess { def rows = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def set = new HashSet() + def index = 0 for(def city : citys) { + index ++ + if(index<145){ + continue + } + println index +" "+city def cityid = cityCodeDatas.get(city) def rets = findPeGteThirdParty2(cityid, DateUtils.formatDateByPattern(new Date(),"yyyyMM"))//当前月日期 println city +" "+rets.size() if(rets){ set.add(city) } - for(def m : rets){ - -// def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}" -// sqlInstance236.executeUpdate(sql) - def querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}" - def total = 0 - pgsqlInstance.eachRow(querySql, { r -> - total = r.count - }) - if(total>0){ - println city +" "+m.commid+" (按价格)删除:"+total - def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}" - pgsqlInstance.executeUpdate(sql) - }else{ - - querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} " - pgsqlInstance.eachRow(querySql, { r -> - total = r.count - }) - if(total>0){ - println city +" "+m.commid+" (按小区)删除:"+total - def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} " - pgsqlInstance.executeUpdate(sql) - } - } - } + processDetele(city, cityid, month, rets) } println set.toString() } + + def processDetele(city, cityid, month, rets){ + + final CountDownLatch latch = new CountDownLatch(rets.size()) + + for(def i=0;i + total = r.count + }) + if(total>0){ + println city +" "+m.commid+" (按价格)删除:"+total + def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}" + pgsqlInstance.executeUpdate(sql) + }else{ + + querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} " + pgsqlInstance.eachRow(querySql, { r -> + total = r.count + }) + if(total>0){ + println city +" "+m.commid+" (按小区)删除:"+total + def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} " + pgsqlInstance.executeUpdate(sql) + } + } + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + } + }); + } + + latch.await() + } + + + + + def findThirdCount(sqlInstance236, sql){ def count = 0 sqlInstance236.eachRow(sql,{r->