From 94889395254ead1eb98ba7c0973dd75c8076ff3e Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Mon, 21 Oct 2024 15:11:36 +0800 Subject: [PATCH 01/35] =?UTF-8?q?=E5=BB=BA=E8=A1=8C=E7=B1=BB=E5=9E=8B?= =?UTF-8?q?=E5=AD=97=E6=AE=B5=E6=94=BE=E6=9C=80=E5=90=8E=E4=B8=80=E5=88=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ronghe/ExportDistrictRongHe.groovy | 80 +++++++++---------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index c611aa10f..1521d4e0b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -1457,45 +1457,45 @@ class ExportDistrictRongHe extends BaseBatch{ println "community_p_describe_gj 上传Ftp完成......" } - def exportGuideStatistics(time){ - - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat" - if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat" - } - def headers = [] - def allRows = [] - - for(def city : citys) { - def cityCode = cityCodeMap.get(city) - def sql = null; - if(CcbCommonUtils.citys_92.contains(city)){ - sql = "select MODEL_TYPE,CITYID,CITYNAME,YEAR_MONTH,COMM_SUM_COUNT,COMM_COUNT,RATIO,TYPE from GUIDE_STATISTICS where cityid='${cityCode}' and year_month=${time}"; - }else{ - sql = "select MODEL_TYPE,CITYID,CITYNAME,YEAR_MONTH,COMM_SUM_COUNT,COMM_COUNT,RATIO,TYPE from GUIDE_STATISTICS where cityid='${cityCode}'"; - } - - sqlInstance.eachRow(sql, { r -> - def data = [] - data << r.MODEL_TYPE - data << r.CITYID - data << r.CITYNAME - data << r.YEAR_MONTH - data << r.COMM_SUM_COUNT - data << r.COMM_COUNT - data << r.RATIO - data << r.TYPE - allRows << data - }); - } - println allRows.size() - FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); - - println "guide_statistics 数据文件输出完成......" - CcbFtpUtils.ftpUpload("rh", fileName, curDate) - println "guide_statistics 上传Ftp完成......" - } +// def exportGuideStatistics(time){ +// +// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" +// def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat" +// if(CommonDatautils.isWindow()){ +// fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat" +// } +// def headers = [] +// def allRows = [] +// +// for(def city : citys) { +// def cityCode = cityCodeMap.get(city) +// def sql = null; +// if(CcbCommonUtils.citys_92.contains(city)){ +// sql = "select MODEL_TYPE,CITYID,CITYNAME,YEAR_MONTH,COMM_SUM_COUNT,COMM_COUNT,RATIO,TYPE from GUIDE_STATISTICS where cityid='${cityCode}' and year_month=${time}"; +// }else{ +// sql = "select MODEL_TYPE,CITYID,CITYNAME,YEAR_MONTH,COMM_SUM_COUNT,COMM_COUNT,RATIO,TYPE from GUIDE_STATISTICS where cityid='${cityCode}'"; +// } +// +// sqlInstance.eachRow(sql, { r -> +// def data = [] +// data << r.MODEL_TYPE +// data << r.CITYID +// data << r.CITYNAME +// data << r.YEAR_MONTH +// data << r.COMM_SUM_COUNT +// data << r.COMM_COUNT +// data << r.RATIO +// data << r.TYPE +// allRows << data +// }); +// } +// println allRows.size() +// FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); +// +// println "guide_statistics 数据文件输出完成......" +// CcbFtpUtils.ftpUpload("rh", fileName, curDate) +// println "guide_statistics 上传Ftp完成......" +// } def exportPgGuideStatistics(time){ @@ -1514,7 +1514,6 @@ class ExportDistrictRongHe extends BaseBatch{ pgsqlInstance.eachRow(sql, { r -> def data = [] - data << r.gujia_model_type data << r.urbn_id data << r.urbn_nm data << r.gujia_clc_yrmo @@ -1522,6 +1521,7 @@ class ExportDistrictRongHe extends BaseBatch{ data << r.gujia_comm_count data << r.gujia_ratio data << r.gujia_type + data << r.gujia_model_type allRows << data }); } From 2f8d203a7079af66e7868c42aed5e094b242d354 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 22 Oct 2024 15:14:16 +0800 Subject: [PATCH 02/35] =?UTF-8?q?=E5=88=A0=E9=99=A4=E9=87=8D=E5=A4=8D?= =?UTF-8?q?=E5=B0=8F=E5=8C=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/gj/district/RemoveDistrict.groovy | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/RemoveDistrict.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/RemoveDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/RemoveDistrict.groovy new file mode 100644 index 000000000..2f49dc227 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/RemoveDistrict.groovy @@ -0,0 +1,56 @@ +package com.fangjia.script.gj.district + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DbUtils +import com.mongodb.BasicDBObject + +/** + * 临时移除 相同名称即在正常状态下有,在删除状态下也有的小区 + */ +class RemoveDistrict { + + public static void main(String[] args) { + def task = new RemoveDistrict() + task.run() + } + + def run(){ + def citys = CommonUtils.findAllCity() + def cat = "shop" + def keys = [] + citys.each {c-> + def districts = CommonDatautils.findAllDistricts(c, cat, ["city", "region", "name", "status"]) + districts.each {it-> + def key = it.city +"##"+it.region+"##"+it.name + if(!keys.contains(key)){ + keys << key + }else{ + println it.toString()+" 删除" + DbUtils.seaweed.remove(new BasicDBObject("_id",it._id)) + } +// def list = CommonDatautils.findAllStatusDistrict(it.city, it.region, it.name, cat, ["city", "region", "name", "status"]) +// if(list && list.size()>1){ +// println it.toString() + " "+list.size() +// for(def d : list){ +//// if(d.status.toString()=="0"){ +//// println d.toString()+" 保留" +//// }else if(d.status.toString()=="1"){ +//// println d.toString()+" 删除" +//// DbUtils.seaweed.remove(new BasicDBObject("_id",d._id)) +//// } +// +// if(d.status.toString()=="1"){ +// println d.toString()+" 保留" +// }else if(d.status.toString()!="1"){ +// println d.toString()+" 删除" +// DbUtils.seaweed.remove(new BasicDBObject("_id",d._id)) +// } +// } +// +// println "................................................................................................" +// } + } + } + } +} From 352c454b2c14d98d92ab27c58ede818af9c32663 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 22 Oct 2024 15:14:41 +0800 Subject: [PATCH 03/35] =?UTF-8?q?=E5=B0=8F=E5=8C=BA=E5=9B=BE=E7=89=87?= =?UTF-8?q?=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/images/AddDistImagesBatch.groovy | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/images/AddDistImagesBatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/images/AddDistImagesBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/images/AddDistImagesBatch.groovy new file mode 100644 index 000000000..9ba1fe9ca --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/images/AddDistImagesBatch.groovy @@ -0,0 +1,146 @@ +package com.fangjia.script.gj.images + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.bson.types.ObjectId +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 更新小区图片 + */ +@BatchOwner(Watcher.GaoJuan) +class AddDistImagesBatch extends BaseBatch{ + final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018) + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + + public static final DBCollection seaweed_fs_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_images", "mongo.db.fangjia.com") + + static main(args) { + runBatch(new AddDistImagesBatch(), args); + } + + @Override + public void process(Object args) { +// def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd") + def mdate = DateUtils.getFirstDayOfMonth(new Date(), -1) + def citys = CommonUtils.findAllCity() + citys.each {city-> + def communitys = findAllEsData2(city, mdate) + println "抓取城市: "+city +" 第三方小区有图片数:"+communitys.size() + //第三方匹配小区 + matchCommunitys(communitys) + } + } + + + + def findAllEsData2(city, mdate){ + QueryBuilder date_range = QueryBuilders.rangeQuery("m_date").from(mdate.getTime()).includeLower(true) + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("source", "安居客小程序")) + .must(date_range).must(date_range) + + def rets = [] + def datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "community", queryBuilder); + if(datas) { + for (int k = 0; k < datas.size(); k++) { + def d = datas[k]; + if(d.house_type_images_2 || d.outdoor_images) { + def data = new BasicDBObject() + data.put("city", d.city) + data.put("region", d.region) + data.put("name", d.name) + data.put("id", d.id) + data.put("out_images", d.outdoor_images) + data.put("house_type_images", d.house_type_images_2) + rets << data + } + } + } + return rets; + } + + def findAllEsData(city, mdate){ + def names = new HashSet() + def rets = [] + ElasticUtils.executeSqlQuery("select * from community where source='安居客小程序' and city='${city}' and m_date>='${mdate}' order by m_date", { d-> + if(d.house_type_images_2 || d.outdoor_images){ + def s = d.city +"#"+d.region+"#"+d.name+"#"+d.property_type + if(!names.contains(s)){ + names << s + def data = new BasicDBObject() + data.put("city", d.city) + data.put("region", d.region) + data.put("name", d.name) + data.put("out_images", d.outdoor_images) + data.put("house_type_images", d.house_type_images_2) + rets << data + } + } + }) + return rets; + } + + + def matchCommunitys(communitys){ + def addCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(communitys.size()) + def rets = new CopyOnWriteArrayList() + def total = communitys.size() + for(def i=0;i Date: Tue, 22 Oct 2024 15:15:34 +0800 Subject: [PATCH 04/35] =?UTF-8?q?=E6=B7=B1=E5=9C=B3=E6=88=90=E4=BA=A4?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/ccbevaluation/AddSzDealData.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy index 974df17ee..0bf6ec4b9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy @@ -9,7 +9,7 @@ import com.mongodb.DBCollection /** * 深圳公众号成交数据录入 */ -def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年7月深圳二手房成交.xlsx", 0, 1) +def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年9月深圳二手房成交.xlsx", 0, 1) def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) From 2701a125d80f3473f37a94eb08e0f42f963f222a Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 22 Oct 2024 15:16:56 +0800 Subject: [PATCH 05/35] =?UTF-8?q?=E6=96=B0=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index f9d5867e0..d8e78df14 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -40,7 +40,7 @@ class CcbCommonUtils { public static final DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com") - +// public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "deal_price_all", "mongo.db.fangjia.com") public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017) From 40a338302d50e4bd4ceca3951c7e8152e40f2748 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 22 Oct 2024 15:17:50 +0800 Subject: [PATCH 06/35] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E8=A1=A5=E5=85=85?= =?UTF-8?q?=E5=9F=8E=E5=B8=82=EF=BC=8C=E5=8C=BA=E5=9F=9F=EF=BC=8C=E5=B0=8F?= =?UTF-8?q?=E5=8C=BA=E5=9D=87=E4=BB=B7=E5=92=8C=E7=A7=9F=E9=87=91=E7=9A=84?= =?UTF-8?q?=E5=91=BD=E4=BB=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/data_handle/AddCityRegionPrice.groovy | 115 +++++++++++++++++- 1 file changed, 112 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy index cc42938d7..954e0402b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy @@ -6,21 +6,33 @@ import com.fangjia.script.averager.InsertMonitorSquareRentCalcCommandBatch import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.MongoClient +import com.rabbitmq.client.AMQP class AddCityRegionPrice { public final static DBCollection COMMAND_COLL = new MongoClient("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('calc_command') + + public static final def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017"; + public static final DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName2("fangjia", "sqmetre_rent", mongoDbUrl) + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", mongoDbUrl) + static main(args) { def task = new AddCityRegionPrice() // task.addRegionPrice() -// task.addCityprice() + task.addCityprice() // task.addDistrictprice() // task.addCityRentPrice() - task.addDistrictRentprice() +// task.addRegionRentPrice() +// task.addDistrictRentprice() +// task.updateRegionRentData() +// task.updateDealPriceAreaType() +// task.checkCityImport() } def addRegionPrice(){ @@ -54,7 +66,7 @@ class AddCityRegionPrice { def addCityprice(){ InsertMonitorCalcCommandBatch insertBatch = new InsertMonitorCalcCommandBatch() - def citys = CommonDatautils.findAllCity().reverse() + def citys = ["乐东"]// CommonDatautils.findAllCity().reverse() def add = 0 for(def c : citys){ def startObj = CommonDatautils.queryCityOrRegionPrice(c, null, 1) @@ -125,6 +137,35 @@ class AddCityRegionPrice { } } + def addRegionRentPrice(){ + InsertMonitorSquareRentCalcCommandBatch inserBatch = new InsertMonitorSquareRentCalcCommandBatch(); + def citys = CommonDatautils.findAllCity() + for(def c : citys){ + + def regions = CommonDatautils.getCityRegions(c) + for(def r : regions){ +// if(r!="南川"){ +// continue +// } + def startObj = CommonDatautils.queryFirstOrLastRentPrice(c, r, null, 1) + def endObj = CommonDatautils.queryFirstOrLastRentPrice(c, r, null, -1) + if(startObj && endObj){ + def months = DateUtils.getBetweenMonth(DateUtil.formatByPattern(startObj.weekend, "yyyyMM"),"202409") + months.each {m-> +// println m + def lastDay = DateUtils.truncateDay(DateUtils.getLastDayOfMonth(DateUtil.parseDate(m+"01", "yyyyMMdd"))) + def obj = CommonDatautils.findSqmetreRent(c, r, null, lastDay) + if(!obj){ + inserBatch.saveCityCalcCommand(c, r, null, lastDay, "fangjia-priceanalysis-0") + println "[add] "+c+" "+r+" "+DateUtil.formatByPattern(lastDay, "yyyy-MM-dd") + } + } + } + } + + } + } + def addDistrictRentprice(){ InsertMonitorSquareRentCalcCommandBatch inserBatch = new InsertMonitorSquareRentCalcCommandBatch(); def citys = CommonDatautils.findAllCity() @@ -158,4 +199,72 @@ class AddCityRegionPrice { } } + + def updateRegionRentData(){ + def q = new BasicDBObject("cat", "region") + def keys = new BasicDBObject() + keys.put("markup", 1) + def dbcursor = SQMETRE_RENT_COLL.find(q, keys).batchSize(5000) + def index = 1 + for(def d : dbcursor){ + if(!(d.markup instanceof Double)){ + def udata = new BasicDBObject() + double markup = 1.0 + markup = d.markup * markup + udata.put("markup", markup) + SQMETRE_RENT_COLL.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + println index + index++ + } + } + } + + + def updateDealPriceAreaType(){ + def index = 1 + def citys = ["凉山州","福州","南京","佛山","沈阳","苏州"] + citys.each {c-> + def q = new BasicDBObject() + q.put("city", c) +// q.put("price_flag", "国信达成交-推导") + def dbcursor = DEAL_PRICE_COLL.find(q) + for(def d : dbcursor){ +// if(!(d.area instanceof Double)){ +// def udata = new BasicDBObject() +// double area = 1.0 +// area = d.area * area +// udata.put("area", area) +// DEAL_PRICE_COLL.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) +// println c+" "+index +// index++ +// } + if(!(d.total_price instanceof Integer)){ + def udata = new BasicDBObject() + udata.put("total_price", d.total_price.toString().toDouble().toInteger()) + DEAL_PRICE_COLL.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + println c+" "+index + index++ + } + } + } + } + + def checkCityImport(){ + def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"] + for(def c : citys){ + def startObj = CommonDatautils.findStatIndexBySort("城市", c, null, 1) + def endObj = CommonDatautils.findStatIndexBySort("城市", c,null, -1) + if(startObj && endObj){ + def months = DateUtils.getBetweenMonth(startObj.yearmonth.toString(),"202409") + months.each {m-> + def obj = CommonDatautils.findStatIndexByMonth("城市", c,null, m.toInteger()) + if(!obj){ + println "[add] "+c+" "+m + } + } + }else{ + println c +" 一个也没有" + } + } + } } From d6a12be1afe39ac35a652d1abb7318814847ff78 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 22 Oct 2024 15:20:52 +0800 Subject: [PATCH 07/35] =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/gj/poc/AddPriceTest.groovy | 36 ++++++++++++++++--- .../thirdPriceCrawl/DataStandardBatch.groovy | 2 +- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 4 +++ 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy index 14b6a5c26..eed8d9fda 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy @@ -25,6 +25,10 @@ class AddPriceTest { def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse", "postgres", "postgres", "org.postgresql.Driver") + //235为房价网内部计算均价的库 + def pgsqlInstance235 = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified", + "postgres", "Goojia123456", "org.postgresql.Driver") + static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") public static final DBCollection confirm_price_date = MongoUtils.getCollectionByName("fangjia", "confirm_price_data", "mongo.db.fangjia.com") @@ -33,7 +37,7 @@ class AddPriceTest { task.run(); } def run(){ - def filePath = "D:\\数据\\建行\\房产估值\\20240716-20240815 _业务反馈价格.xlsx" + def filePath = "D:\\数据\\建行\\房产估值\\建行业务反馈badcase.xlsx" // addGXDPrice(filePath) addCcbPrice(filePath) // addThirdPrice(filePath) @@ -173,17 +177,22 @@ class AddPriceTest { def cityId = cityCodeDataMap.get(d['V']) def map = findAvgPrice(d['U'], cityId) if(map){ - d['F'] = map.avg_price -// d['P'] = map.calc_type + d['H'] = map.avg_price + d['I'] = map.calc_type // d['AM'] = map.credit_value } + +// def newPriceMap = findNewFjwAvgPrice(d['U'], cityId) +// if(newPriceMap){ +// d['Q'] = newPriceMap.avg_price +// } } }) } def findAvgPrice(def commid, def cityid){ - def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202407 order by gujiabds_clc_yrmo_val desc limit 1" + def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202409 order by gujiabds_clc_yrmo_val desc limit 1" def map = [:] pgsqlInstance.eachRow(sql, { r -> map.avg_price = r.gujia_bds_clc_avprc @@ -202,6 +211,25 @@ class AddPriceTest { } + def findNewFjwAvgPrice(def commid, def cityid){ + def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from price_stat.avm_community_base_price where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202409 order by gujiabds_clc_yrmo_val desc limit 1" + def map = [:] + pgsqlInstance235.eachRow(sql, { r -> + map.avg_price = r.gujia_bds_clc_avprc + map.type = r.gujia_bds_clmtd + map.month = r.gujiabds_clc_yrmo_val + map.credit_value = r.gujia_bds_cfdc_dgr + if(r.gujia_bds_clmtd == 1){ + map.calc_type = "人工估值" + }else if(r.gujia_bds_clmtd == 2){ + map.calc_type = "样本计算" + }else if(r.gujia_bds_clmtd == 3){ + map.calc_type = "神经网络" + } + }) + return map + } + def addThirdPrice(filepath){ ExcelUtils.multiThreadingModify(filepath, { d -> if (!d['X']) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index 1d63a790f..b6e1b0ced 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -62,7 +62,7 @@ class DataStandardBatch extends BaseBatch{ private static def districtMap = [:]; private static def limit = 5000; - def static final SHORT_DATE = Integer.parseInt(com.fangjia.script.util.DateUtils.addMonth(com.fangjia.script.util.DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); + def static final SHORT_DATE = Integer.parseInt(com.fangjia.script.util.DateUtils.addMonth(com.fangjia.script.util.DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6, "无名称":7] def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋","幸福里","中国房价行情","常州住房"] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 668fcbb74..5f07adb27 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -76,6 +76,10 @@ class SynchDistrictPrice extends BaseBatch{ data.put("build_type",d.build_type) data.put("c_date", new Date()) data.put("m_date", new Date()) +// if(d.location){ +// data.put("lng", d.location.x) +// data.put("lat", d.location.y) +// } rets << data } }) From c859830a101ff263d5f0c0503ed8525c5d65a713 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 22 Oct 2024 15:21:47 +0800 Subject: [PATCH 08/35] =?UTF-8?q?=E7=AC=AC=E4=B8=89=E6=96=B9=E5=9D=87?= =?UTF-8?q?=E4=BB=B7=E8=A1=A8=E4=BF=AE=E6=94=B9=E5=8C=BA=E5=9F=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/data_handle/RegionUpdateBatch.groovy | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionUpdateBatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionUpdateBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionUpdateBatch.groovy new file mode 100644 index 000000000..6fce88fa6 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionUpdateBatch.groovy @@ -0,0 +1,65 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +//更新第三方均价表中小区区域 +class RegionUpdateBatch { + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50) + + static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + + static main(args) { + def task = new RegionUpdateBatch() + task.run() + + } + def run(){ + def citys = CommonDatautils.findAllCity() + for(def city : citys){ + def districtMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["city","region", "name"]) + def keys = new BasicDBObject() + keys.put("city", 1) + keys.put("region", 1) + keys.put("name", 1) + keys.put("seaweed_id", 1) + def statistics = AVERAGE_STATISTICS.find(new BasicDBObject("city", city), keys) + def total = statistics.size() + final CountDownLatch latch = new CountDownLatch(total) + def totalCount = new AtomicLong() + for(def i=0;i Date: Tue, 22 Oct 2024 15:22:23 +0800 Subject: [PATCH 09/35] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=B0=8F=E5=8C=BA?= =?UTF-8?q?=E5=9B=B4=E6=A0=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../data_handle/AddDistrictShapeBatch.groovy | 286 ++++++++++++++++++ 1 file changed, 286 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy new file mode 100644 index 000000000..5093ecb3a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy @@ -0,0 +1,286 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.rabbitmq.client.impl.AMQImpl + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 添加高德地图,百度地图围栏 + */ +@BatchOwner(Watcher.GaoJuan) +class AddDistrictShapeBatch extends BaseBatch{ + + def fullCityMap = CommonDatautils.queryFullNameCity() + def mcaDistMap = CommonDatautils.queryDistNameCityByMapping() + + DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020) + public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com") + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) + + static main(args) { + runBatch(new AddDistrictShapeBatch(), args) + } + + @Override + public void process(Object args) { + def citys = CommonDatautils.findAllCity() + def index = 1 + for(def c : citys){ + println c +" "+index + index ++ + def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2"]) +// baiduMatch(c, districts) + gaodeMatch(c, districts) + + } + } + + def baiduMatch(city,districts){ + def data = fullCityMap.get(city) + def fullCityName = null + if(data){ + fullCityName = data.city + } + final CountDownLatch latch = new CountDownLatch(districts.size()) + + if(fullCityName){ + def rets = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + for(def i=0;i1) { +// resultMap.type = "MultiPolygon" +// }else{ + resultMap.type = "polygon" +// } + o.put("shape", resultMap) + + def q = new BasicDBObject("comm_id", d._id.toString()) + seaweed_fence_coll.update(q, new BasicDBObject('$set', o),true,false) +// seaweed_fence_coll.save(o) + println "[save] 围栏 ( "+d.city +" "+d.region +" "+d.name+" )" + return fenceMap.data_id + } + return null + } + + def findBaiduDatas(city, district, name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("district", district) + q.put("name", name) + q.put('std_tag', '房地产;住宅区') + q.put("guoke_geo", new BasicDBObject('$ne', null)) + def dbcursor = BAIDU_POI_COLL.find(q) + def map = [:] + println dbcursor.count() + for(def d : dbcursor){ + def arrs = d.gcj02_guoke_geo.split("\\|") + + def coordinatesStr = arrs[2].toString().substring(2, arrs[2].toString().size()-1 ) + // 将字符串分割成列表 + def coordinatePairs = coordinatesStr.split(',') + def coordinatesList = [] + for (int i = 0; i < coordinatePairs.size(); i += 2) { + // 将字符串坐标转换为浮点数 + double longitude = coordinatePairs[i] as double + double latitude = coordinatePairs[i + 1] as double + + // 将坐标点作为一个元组添加到集合中 + coordinatesList << [longitude, latitude] + } + if(coordinatePairs){ + map.data_id = d._id + map.coordinatePairs = coordinatesList + } + } + return map; + } + + + def gaodeMatch(city,districts){ + def data = fullCityMap.get(city) + def fullCityName = null + if(data){ + fullCityName = data.city + } + final CountDownLatch latch = new CountDownLatch(districts.size()) + + if(fullCityName){ + def rets = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + for(def i=0;i + def newName = d.name.replace("-","").replace("·","") + if(newName == name && d.type.contains("住宅") && d.shape){ + obj = d + } + }) + return obj; + } + def findSeaweedFenceExists(commid){ + def q = new BasicDBObject() + q.put("comm_id", commid) + return seaweed_fence_coll.findOne(q) + } + + +} From a1a06d0d8f8a1efe538bda82a592f4dc91c0ebca Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 22 Oct 2024 15:22:57 +0800 Subject: [PATCH 10/35] =?UTF-8?q?=E8=87=AA=E5=8A=A8=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E5=B0=8F=E5=8C=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../data_handle/AutoAddDistrictBatch.groovy | 234 ++++++++++++++++++ 1 file changed, 234 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy new file mode 100644 index 000000000..f27ee361f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy @@ -0,0 +1,234 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.BaseBatch +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 小区自动入库(新增小区,每月运行一次) + */ +class AutoAddDistrictBatch extends BaseBatch{ + + final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018) + public static DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") + DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020) + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + + def fullCityMap = CommonDatautils.queryFullNameCity() + def mcaDistMap = CommonDatautils.queryDistNameCityByMapping() + + static main(args) { + runBatch(new AutoAddDistrictBatch(), args) + } + + @Override + void process(Object args) { + //查找第三方匹配没有匹配上的小区 + def citys = CommonDatautils.findAllCity() + def list = [] + list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "高德地图"] + for(def c : citys){ + def noMatchDatas = findNoMatchResult(c) + println c +" 新旧算法未匹配的数据量:"+noMatchDatas.size() + def rets = checkDatas(noMatchDatas.keySet()) + list.addAll(rets) + } + + def fileName = "/home/goojia/gj/add_dist_info.xlsx" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\数据建设\\名录集\\第三方待入库数据.xlsx" + } + ExcelUtils.write(fileName, list, "待入库数据") + } + + def checkDatas(keysets){ + def noMatchCount = keysets.size() + final CountDownLatch latch = new CountDownLatch(noMatchCount) + + def rets = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + for(def i=0;i + if(d.name == name && d.type.contains("住宅") && d.shape){ + obj = d + } + }) + return obj; + } + + def findTrades(city, region, name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("district_name", name) + return DEAL_PRICE_COLL.count(q) + } + + def findNoMatchResult(def c){ + def q = new BasicDBObject() + q.put("match_id", new BasicDBObject('$exists',false)) + q.put("old_match_id", new BasicDBObject('$exists',false)) + q.put("city", c) + def dbcursor = thirdSeaweedMappingColl.find(q).batchSize(5000) + def rMap = [:] + for(def d : dbcursor){ + if(d.estate_type){ + if(d.estate_type.contains("自建房")){ + continue + } + if(d.estate_type.contains("购物中心")){ + continue + } + if(d.estate_type.contains("住宅底商") || d.estate_type.contains("酒店") || d.estate_type.contains("商铺")){ + continue + } + if(d.estate_type.contains("办公") || d.estate_type.contains("写字楼")){ + continue + } + }else{ + continue + } + def key = d.city + "##"+d.district+"##"+d.name + def rets = rMap.get(key) + if(!rMap.containsKey(key)){ + rets = [] + rMap.put(key, rets) + } + rets << d + } + return rMap + } + + def createData(city, region, name, address, lng, lat){ + def udata = new BasicDBObject() + udata.put("city", city) + udata.put("region", region) + name = name.replaceAll('[·\\(\\)()]|·', '').replace("建设中", "") + udata.put("name", name) + + def addr = AddressUtil.formatAddress(city, address) + if(addr){ + udata.put("address", addr) + } + + udata.put("status", "0") + udata.put("name_cat", "0") + udata.put("cat", "district") + udata.put("username", "高娟") + + def estateType2 = "普通住宅" + def type = "" + if(type){ + if(type.contains("别墅")){ + estateType2 = "别墅" + }else if(type.contains("商住")){ + estateType2 = "商住" + } + } + udata.put("estateType2", estateType2) + if(lng){ + Long gd_lng = lng*10000000000 + udata.put("lng", gd_lng) + } + if(lat){ + Long gd_lat = lat * 10000000000 + udata.put("lat", gd_lat) + } + FangjiaClientUtils.saveOrUpdateDistrict(udata) + } + + def findBaiduDatas(city, district, name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("district", district) + q.put("name", name) + q.put('std_tag', '房地产;住宅区') + def count = BAIDU_POI_COLL.count(q) + return count; + } +} From ea56f1fd1c9dc530cc956470c9b3cadbe7a6b025 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 22 Oct 2024 15:23:50 +0800 Subject: [PATCH 11/35] =?UTF-8?q?=E4=B9=A1=E9=95=87=E8=A1=97=E9=81=93?= =?UTF-8?q?=E4=B8=AD=E5=BF=83=E7=82=B9=E6=A3=80=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/data_handle/TownCenterBatch.groovy | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownCenterBatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownCenterBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownCenterBatch.groovy new file mode 100644 index 000000000..e1bfa8969 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownCenterBatch.groovy @@ -0,0 +1,79 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +/** + * 乡镇街道 判断 中心点是否在围栏或一个围栏是否包含多个中心点 + */ +@BatchOwner(Watcher.GaoJuan) +class TownCenterBatch extends BaseBatch{ + + DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com") + + static main(args) { + runBatch(new TownCenterBatch(), args) + } + + @Override + void process(Object args) { + def shapes = findHasShapeTown() + def centers = findAllCenters() + checkCenter(shapes, centers) + + } + + def checkCenter(shapes, centers){ + def map = [:] + for(def town : shapes){ + def points = InsidePolygon.createPoint4(town.shape.coordinates) + //1.先判断当前镇的中心点是否在围栏中 + if(town.location){ + def flag = InsidePolygon.isInArea(town.location.lng, town.location.lat, points) + if(!flag){ + println town.city + " "+town.district+" "+town.street+" 不在本身围栏中" + } + } + //2.判断是否有其它点在围栏中 + for(def center : centers){ + if(center._id.toString() != town._id.toString()){ + def flag = InsidePolygon.isInArea(center.location.lng, center.location.lat, points) + if(flag){ + def list = map.get(town.city + "#"+town.district+"#"+town.street) + if(!list){ + list = [] + map.put(town.city + "#"+town.district+"#"+town.street) + } + list << center.city + "#"+center.district+"#"+center.street + println center.city + " "+center.district+" "+center.street+" 在 "+town.city + " "+town.district+" "+town.street+" 围栏中" + } + } + } + } + return map + } + + def findHasShapeTown(){ + def q = new BasicDBObject() + q.put("type", "乡镇街道") + q.put("shape", new BasicDBObject('$exists', true)) + return ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(2000).toList() + } + + def findAllCenters(){ + def q = new BasicDBObject() + q.put("type", "乡镇街道") + q.put("location", new BasicDBObject('$exists', true)) + def keys = new BasicDBObject() + keys.put("location", 1) + keys.put("city", 1) + keys.put("district", 1) + keys.put("street", 1) + return ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).toList() + } +} From 1d8e9fa91388974034ff1c9631dfe3a8feb26c3c Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 22 Oct 2024 15:24:55 +0800 Subject: [PATCH 12/35] =?UTF-8?q?=E5=B0=8F=E5=8C=BA=E5=AD=97=E6=AE=B5?= =?UTF-8?q?=E8=87=AA=E5=A1=AB=E5=85=85=E5=A2=9E=E5=8A=A0=E7=BB=8F=E7=BA=AC?= =?UTF-8?q?=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/district/DistrictFieldAddBatch.groovy | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy index 76fa40fb2..3b3aed61a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy @@ -47,7 +47,7 @@ class DistrictFieldAddBatch extends BaseBatch{ //"alias":"alias", //ownership_type:权属类型 为商品房、经适房、动迁房等 //house_property 产权性质:国有产权 "house_property":"网站没有", - def fieldsMap = ["estate_charge":"property_fee","block":"block","build_date":"build_year","estate_company":"property_company","developer":"developer", + def fieldsMap = ["lng":"location","estate_charge":"property_fee","block":"block","build_date":"build_year","estate_company":"property_company","developer":"developer", "virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area","household_count":"households", "car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address","build_structure":"building_structure", "build_type":"build_type", "land_property":"ownership_type","live_date":"live_date", @@ -115,7 +115,7 @@ class DistrictFieldAddBatch extends BaseBatch{ println "【通过匹配填充字段】网站:"+seaweedmapSource +" Es:"+esRets.size()+" 融合小区占比:"+DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/totalCount)+"%" def precentMap = fieldUpdate(totalCount, updateRets) println c +" "+precentMap.toString() - //updateDB(c, updateRets) + updateDB(c, updateRets) println "------------------------------------------------------------------------------" def pre1 = 0 def pre2 = 0 @@ -212,6 +212,11 @@ class DistrictFieldAddBatch extends BaseBatch{ def rets = [] rets << ["第三方城市","第三方区域","第三方名称","第三方地址","第三方户数","匹配区域","匹配名称","匹配可信度","方式"] for(def matchDist : matchRets){ + +// if(!["智富中心","龙山新寓","天怡瀚园","远盛凤林府","京口路小区","梦溪路小区","浩创优尚","溪山丽苑","旭园"].contains(matchDist.name)){ +// continue +// } + def dist = CommonDatautils.findDistrictFieldById(matchDist.match_id, fields) if(!dist){ println c+" "+matchDist.match_id+" 不存在" @@ -306,6 +311,15 @@ class DistrictFieldAddBatch extends BaseBatch{ if(newVal){ newValues.put(seaweedfield, newVal) } + if(seaweedfield == "lng"){ + def locationMap = FiledStrand.processLngAndLat(esVal) + if(locationMap){ + newValues.put("lat", locationMap.lat) + newValues.put("lng", locationMap.lng) + newValues.put("lat2", locationMap.lat2) + newValues.put("lng2", locationMap.lng2) + } + } } } } From e1fbc1ee6a5daea2fea1b11f89a0e7a01b467ee1 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 22 Oct 2024 15:32:18 +0800 Subject: [PATCH 13/35] update --- .../ccbevaluation/AddThirdPriceSample.groovy | 82 +++++++++++-- .../fangjia/script/ccbevaluation/Test.groovy | 100 ++++++++-------- .../script/ccbevaluation/TradeStatLog.groovy | 21 +++- .../ccbevaluation/data/AddressCut.groovy | 97 +++++++++------ .../script/commData/CommonDatautils.groovy | 75 ++++++++++-- .../com/fangjia/script/gj/FiledStrand.groovy | 24 ++++ .../com/fangjia/script/gj/TradeMatch.groovy | 96 ++++++++++++--- .../script/gj/data_handle/DevProcess.groovy | 4 +- .../gj/data_handle/DevUpdateTestBatch.groovy | 10 +- .../gj/data_handle/GXDdistrictMatch.groovy | 28 ++++- .../data_handle/ThirdPriceMappingBatch.groovy | 9 +- .../gj/data_handle/TownMiddleProcess.groovy | 112 +++++++++++++----- .../script/trade/ImportIndexStats.groovy | 85 +++++++++---- 13 files changed, 556 insertions(+), 187 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPriceSample.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPriceSample.groovy index 2684f90c2..a36d9d715 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPriceSample.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPriceSample.groovy @@ -13,19 +13,24 @@ class AddThirdPriceSample { static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + public static final DBCollection NO_PRICE_DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "deal_price_all", "mongo.db.fangjia.com") + + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") + static main(args) { def task = new AddThirdPriceSample() - task.run() + task.run() //添加第三方 fccs->保存的是国信达的参考价格 csjj->保存的是国信达的成交数据 +// task.addTrade() //添加成交 } def run(){ - for(def i=1;i<19;i++){ - ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\级联待优化小区集合_7月.xlsx",{d-> - if(d['A'] && d['C']){ + for(def i=0;i<19;i++){ + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\国信达参考价作为第三方入库.xlsx",{d-> + if(d['A'] && d['B']){ def dist = CommonDatautils.findDistrictById(d['A'].toString().trim()) if(dist){ - def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202407) - def udata = new BasicDBObject("csjj_esf_price", d['C'].toString().toDouble().toInteger()) - + def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202408) +// def udata = new BasicDBObject("csjj_esf_price", d['C'].toString().toDouble().toInteger()) + def udata = new BasicDBObject("fccs_esf_price", d['B'].toString().toDouble().toInteger()) AVERAGE_STATISTICS.update(q,new BasicDBObject('$set', udata),false, false) }else{ println d['A'] +" 没查到" @@ -37,4 +42,67 @@ class AddThirdPriceSample { } } + + def addTrade(){ + def citys = ["武汉","南京","杭州","长沙","郑州","合肥","宁波","南通","嘉兴","珠海","泉州","绍兴","湖州","镇江"] + def rets = findTrades(citys) +// DEAL_PRICE_COLL.insert(rets) + } + + def findTrades(citys){ + def q = new BasicDBObject() + q.put("city", new BasicDBObject('$in', citys)) + q.put("flag", 1) + def dbcursor = NO_PRICE_DEAL_PRICE_COLL.find(q) + def rets = [] + def index = 0 + for(def db : dbcursor){ + index++ + def udata = new BasicDBObject() + + udata.put("_id",db._id) + udata.put("url",db.url) + udata.put("area",db.area) + udata.put("avg_price",db.avg_price.toString().toDouble().toInteger()) + udata.put("build_structure",db.build_structure) + udata.put("build_type",db.build_type) + udata.put("build_year",db.build_year) + udata.put("city",db.city) + udata.put("direction", db.direction) + udata.put("district_name",db.district_name) + udata.put("elevator",db.elevator) + udata.put("fitment",db.fitment) + udata.put("floor_info",db.floor_info) + udata.put("hall",db.hall.toString().toInteger()) + udata.put("height",db.height.toString().toInteger()) + udata.put("housing_age",db.housing_age) + udata.put("lift",db.lift.toString().toInteger()) + udata.put("lift_household",db.lift_household.toString().toInteger()) + udata.put("property_ownership",db.property_ownership) + udata.put("region",db.region) + udata.put("room", db.room.toString().toInteger()) + udata.put("room_structure",db.room_structure) + udata.put("source",db.source) + udata.put("total_price",db.total_price.toString().toInteger()) + udata.put("trade_date",db.trade_date) + udata.put("trade_property",db.trade_property) + udata.put("house_using",db.house_using) + udata.put("heating_type",db.heating_type) + udata.put("community_id",db.community_id) + udata.put("create_date",db.create_time) + udata.put("m_date", new Date()) + udata.put("address",db.address) + udata.put("match_city",db.match_city) + udata.put("match_region",db.match_region) + udata.put("match_name",db.match_name) + udata.put("match_id",db.match_id) + udata.put("price_flag", "国信达成交-推导") + + DEAL_PRICE_COLL.save(udata) + println index +// rets << udata + } + return rets + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 0d36523e6..fed8e9757 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -631,36 +631,36 @@ def findAllData(sqlInstance236, citycode){ //},0) -ExcelUtils.modify("D:\\数据\\建行\\房产估值\\添加别名.xlsx",{d-> - if(d['A'] && d['B']){ - def dbo = CommonDatautils.findDistrictById(d['A']) - if(dbo){ - def alias = null - if(!dbo.alias){ - alias = [] - }else{ - alias = dbo.alias - } - if(!alias.contains(d['B'])){ - alias << d['B'] - } - if(alias){ - def udata = new BasicDBObject() - udata.put("alias", alias) - udata.put("m_date",new Date()) - - def q = new BasicDBObject() - q.put("_id", dbo._id) - DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) - }else{ - println d['A']+" "+"别名已存在" - } - - }else{ - println dbo._id.toString() +" 没找到" - } - } -}) +//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\添加别名.xlsx",{d-> +// if(d['A'] && d['B']){ +// def dbo = CommonDatautils.findDistrictById(d['A']) +// if(dbo){ +// def alias = null +// if(!dbo.alias){ +// alias = [] +// }else{ +// alias = dbo.alias +// } +// if(!alias.contains(d['B'])){ +// alias << d['B'] +// } +// if(alias){ +// def udata = new BasicDBObject() +// udata.put("alias", alias) +// udata.put("m_date",new Date()) +// +// def q = new BasicDBObject() +// q.put("_id", dbo._id) +// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +// }else{ +// println d['A']+" "+"别名已存在" +// } +// +// }else{ +// println dbo._id.toString() +" 没找到" +// } +// } +//}) //def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"] //for(def c : citys){ @@ -1331,34 +1331,34 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ -//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240806--国信达价格录入.xlsx" +def fileName = "D:\\数据\\建行\\房产估值\\6个城市待优化_9月xlsx.xlsx" // -//ExcelUtils.multiThreadingModify(fileName, {d-> -// if(d['J']){ -//// def region = RegionData.formatRegion(d['A'], d['J']) -//// if(region){ -//// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district") -//// if(dist){ -//// d['B'] = dist.region -//// d['C'] = dist.name -//// d['L'] = dist.status-dist.name_cat -//// } -//// } -//// if(d['J'] instanceof String){ -// def dist = CommonDatautils.findDistrictById(d['J']) +ExcelUtils.multiThreadingModify(fileName, {d-> + if(d['G']){ +// def region = RegionData.formatRegion(d['A'], d['J']) +// if(region){ +// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district") // if(dist){ +// d['B'] = dist.region +// d['C'] = dist.name +// d['L'] = dist.status-dist.name_cat +// } +// } + if(d['G'] instanceof String){ + def dist = CommonDatautils.findDistrictById(d['G']) + if(dist){ // d['A'] = dist.city // d['B'] = dist.region // d['C'] = dist.name // d['K'] = dist.estate_type2 // d['L'] = dist.status // d['M'] = dist.name_cat -// -//// d['H'] = dist.name -// } -//// } -// } -//},0) + + d['H'] = dist.name + } + } + } +},0) //def file = "D:\\数据\\建行\\房产估值\\指导价\\5城价格翻新6-21.xlsx" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index 9fc014cca..c2caf475f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -47,7 +47,8 @@ class TradeStatLog { // task.run()//统计最近3个月抓取的成交数据量 // task.exportTrade()//导出未入库成交 - task.exportTrade2()//导出未入库成交 +// task.exportTrade2()//导出未入库成交 + task.updateDealPriceRegion()//修改成交区域数据 // task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) // task.averageStat() @@ -597,7 +598,7 @@ class TradeStatLog { // def endDate = "2024-07-01" def ids = findAllDealIds(); println ids.size() - ExcelUtils.multiThreadingModify("D:\\数据\\其它\\2024年6月成交数据.xlsx",{d-> + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\建行成交\\廊坊.xlsx",{d-> if(ids.contains(d['A'])){ d['V'] = "供" } @@ -674,7 +675,7 @@ class TradeStatLog { } def findAllDealIds(){ - def sql = "select gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where p9_start_date='2024-07-28'" + def sql = "select gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where urbn_nm='廊坊'" def ids = [] pssqlInstance.eachRow(sql,{r-> if(!ids.contains(r.gujia_bds_mdl_id)){ @@ -801,6 +802,20 @@ class TradeStatLog { },0) } + def updateDealPriceRegion(){ + ExcelUtils.modify("D:\\数据\\建行\\建行成交\\成交待修改区域数据.xlsx",{d-> + if(d['A'] && d['G']){ + def region = RegionData.formatRegion(d['D'], d['G']) + if(region){ + def udata = new BasicDBObject() + udata.put("region", region) + DEAL_PRICE_COLL.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata)) + } + + } + },0) + } + def findTradeById(id){ def q = new BasicDBObject() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy index af276a9c9..10a3cc392 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy @@ -24,6 +24,13 @@ class AddressCut extends BaseBatch{ final DBCollection thirdPartyPriceColl = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018) + final DBCollection thirdPartyPriceBkColl = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price_history_bk_new", "192.168.0.235", 27018) + + + final DBCollection dataProcessColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process", "192.168.0.235", 27018) + + final DBCollection dataProcessBkColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process_history_bk", "192.168.0.235", 27018) + public static EsDataSource dataSource = new EsDataSource() private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000" static { @@ -39,50 +46,66 @@ class AddressCut extends BaseBatch{ @Override public void process(Object args) { -<<<<<<< HEAD // sameAddr() - seaweedStreetUpdate() + def dates = [202408] + for(def date : dates){ +// synch(date) + synchDataProcess(date) + println date + "同步完成! " + } + } - def seaweedStreetUpdate(){ - ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区地址拆分\\已核实-路2.xlsx",{d-> - if(d['A']){ - def dbo = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district") - if(dbo){ - def udate = new BasicDBObject() - def street = null - def address = "" - if(d['E']){ - street = d['E'] - udate.put("street", street) - address = street - } - def streetNum = null - if(d['F']){ - streetNum = d['F'] - udate.put("street_number", streetNum) - address = address + streetNum - } - if(dbo.address!=address){ - udate.put("address", address) - } - if(udate){ - udate.put("m_date", new Date()) - DBUtils.seaweed.update(new BasicDBObject('_id', dbo.get("_id")), new BasicDBObject('$set', udate)) - } - } + def synch(date){ + def q = new BasicDBObject() + q.put("s_date", date) + def dbcursor = thirdPartyPriceColl.find(q).batchSize(5000) + def index = 0 + def rets = [] + def num = 0 + for(def d : dbcursor){ + num++ + index++ + rets << d + if(index%10000==0){ + thirdPartyPriceBkColl.insert(rets) + println(num) + rets.clear() } - - }) -======= -// sameAddr() //相同地址小区 -// roadMatch() - addAnjukeRoad() - + } + if(rets.size()>0){ + thirdPartyPriceBkColl.insert(rets) + rets.clear() + } } + def synchDataProcess(date){ + def q = new BasicDBObject() + q.put("s_date", date) + def dbcursor = dataProcessColl.find(q).batchSize(5000) + def index = 0 + def rets = [] + def num = 0 + for(def d : dbcursor){ + num++ + index++ + rets << d + if(index%10000==0){ + dataProcessBkColl.insert(rets) + println(num) + rets.clear() + } + } + if(rets.size()>0){ + dataProcessBkColl.insert(rets) + rets.clear() + } + } + + + def addAnjukeRoad(){ ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区地址拆分\\小区地址切分(结果)_1.xlsx", { d -> def roads = findThirdPartyPrice(d['A'], d['B'], d['C'], "链家") @@ -130,8 +153,6 @@ class AddressCut extends BaseBatch{ } }) - ->>>>>>> c1a5ab8c6aa973a91e6be5547ec0925db3fa500f } def sameAddr() { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 94668a4df..5e2937768 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -122,6 +122,8 @@ class CommonDatautils { public static final DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com") public static final DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com") + public static final DBCollection STATS_INDEX_COLL = MongoUtils.getCollectionByName("fangjia_stat", "stats_index", "mongo.db.fangjia.com") + public static DSCollection getHouseDataStoreConnection(){ if(house_dataStoreConnection != null) { return house_dataStoreConnection @@ -820,6 +822,7 @@ class CommonDatautils { DBObject q = new BasicDBObject(); q.put("status",0) +// q.put("status",new BasicDBObject('$ne', 0)) q.put("cat", cat); q.put("city", city) @@ -829,7 +832,7 @@ class CommonDatautils { keys.put(field, 1); } } - def dbsour = SEAWEED_COLL.find(q, keys).batchSize(2000) + def dbsour = SEAWEED_COLL.find(q, keys).batchSize(5000) def rets = [] for(def d : dbsour){ rets << d @@ -959,13 +962,7 @@ class CommonDatautils { keys.put(field, 1); } } - def cursor = getDataStoreConnection().find(q, keys).batchSize(2000) - if(cursor){ - for(def obj : cursor){ - return obj; - } - } - return null; + return SEAWEED_COLL.findOne(q, keys) } @@ -1132,6 +1129,10 @@ class CommonDatautils { q.put("city", city); q.put("cat", cat); q.put("name", name); + if(region){ + q.put("region", region); + } + q.put("status",new BasicDBObject('$ne', 0)) DBObject keys = new BasicDBObject(); if(fields){ @@ -1680,7 +1681,7 @@ class CommonDatautils { DBObject queryCondition = new BasicDBObject() queryCondition.put("city", city); queryCondition.put("category","region"); - queryCondition.put("show",1); +// queryCondition.put("show",1); def dbCursor = REGION_BLOCK_COLL.find(queryCondition) def regions = []; for(DBObject obj : dbCursor){ @@ -2385,6 +2386,9 @@ class CommonDatautils { q.put("city", city); if(region){ q.put("region", region); + q.put("cat", "区域") + }else{ + q.put("cat", "城市") } q.put('weekend', weekend); def dbCursor = REGION_STAT_COLL.find(q).sort(new BasicDBObject().append("weekend", -1)).limit(1); @@ -3735,9 +3739,9 @@ class CommonDatautils { if(!region){ if(city){ q.put("cat", "city") - }else{ - q.put("cat", "region") } + }else{ + q.put("cat", "region") } } @@ -4095,6 +4099,21 @@ class CommonDatautils { return retMap; } + + static def findHouseCount(def city, def region, def name){ + + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) + .must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("district_name", name)).must(QueryBuilders.termQuery("cat", "sell")) + + + def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"] + def index = new Random().nextInt(5); + def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder); + return houses.size(); + } + + /** * 是否是别墅 * @return @@ -5060,4 +5079,38 @@ class CommonDatautils { } + static def findStatIndexBySort(type, city, region, sort){ + def q = new BasicDBObject() + q.put("cat", "district") + q.put("type", type) + if(region){ + q.put("district", region) + } + if(city){ + q.put("city", city) + } + + def cur = STATS_INDEX_COLL.find(q).sort(new BasicDBObject("yearmonth", sort)).limit(1) + cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def obj = 0; + while(cur.hasNext()) { + obj = cur.next() + } + return obj; + } + + static def findStatIndexByMonth(type, city, region, month){ + def q = new BasicDBObject() + q.put("cat", "district") + q.put("type", type) + if(region){ + q.put("district", region) + } + if(city){ + q.put("city", city) + } + q.put("yearmonth", month) + return STATS_INDEX_COLL.findOne(q) + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy index 1b11ee16a..a3c119bec 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy @@ -2,8 +2,10 @@ package com.fangjia.script.gj import com.fangjia.core.BlockData import com.fangjia.core.RegionData +import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.core.util.DateUtil import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.linxsh.util.AddressUtil import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils @@ -531,6 +533,28 @@ class FiledStrand { return null } + + static def processLngAndLat(val) { + if (val) { + def map = [:] + def lng = DataHandleUtils.getDoubleRound(6, val.x) + def lat = DataHandleUtils.getDoubleRound(6, val.y) + Long gd_lat = lat * 10000000000L + Long gd_lng = lng * 10000000000L + map.lat = gd_lat + map.lng = gd_lng + + def baidu = BaiDuApiUtils.geoconv(lng, lat, 3) + Long bd_lng = baidu.x * 10000000000L + Long bd_lat = baidu.y * 10000000000L + map.lat2 = bd_lat + map.lng2 = bd_lng + return map + + } + return null + } + static def isRoad(def s){ def pattern = Pattern.compile("[路|街|大道|巷]"); def matcher = pattern.matcher(s) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TradeMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TradeMatch.groovy index 082e2f170..a496ed93f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TradeMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TradeMatch.groovy @@ -2,33 +2,97 @@ package com.fangjia.script.gj import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.DataUtil import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection -ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\成交未匹配.xlsx",{d-> - if(d['A'] && !d['E']){ -// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'].toString(), "district") -// def dist = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'].toString(), "district") - println d['A'] +" "+d['C'] -// def total = findDatas(d['A'], d['B'], d['C']) -// d['D'] = total - def c = getAmapLocation(d['A']+"市", d['B']+"区", d['C'].replace("?","").replace("·","").replace("?","").replace("|","").replace("·","")) - d['E'] = c - } -}) +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +//ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\成交未匹配.xlsx",{d-> +// if(d['A'] && !d['E']){ +//// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'].toString(), "district") +//// def dist = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'].toString(), "district") +// println d['A'] +" "+d['C'] +//// def total = findDatas(d['A'], d['B'], d['C']) +//// d['D'] = total +// def c = getAmapLocation(d['A']+"市", d['B']+"区", d['C'].replace("?","").replace("·","").replace("?","").replace("|","").replace("·","")) +// d['E'] = c +// } +//}) + +def URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; +DBCollection seaweed_coll = MongoUtils.getCollectionByName2("fangjia", "seaweed", URL) +def q = new BasicDBObject() +q.put("city", "上海市") +q.put("category", "district") +q.put("status", 0) +def ret = [] +def list = seaweed_coll.find(q).batchSize(5000).toList() +println "总计:"+list.size() +//for(def l : list){ +// def obj = getAmapLocation(l.city, l.region, l.name) +// if(obj){ +// ret << [l.city, l.region, l.name, obj.cityname, obj.adname, obj.name,obj.type] +// } +//} + +ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) +def addCount = new AtomicLong() +final CountDownLatch latch = new CountDownLatch(list.size()) +def total = list.size() +for(def i=0;i - if(d.name.contains(name) && d.type.contains("住宅")){ - total ++ + if(d.name.contains(name) && d.type.contains("住宅") && d.shape){ + obj = d } }) - return total; + return obj; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index 56915936b..f722db94c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -46,7 +46,7 @@ class DevProcess { def task = new DevProcess(); // task.run() // task.parseCityAndDist() - task.devSplitRoad() +// task.devSplitRoad() // task.devNameSplitRoad() // task.addFlag() // task.matchDev() @@ -63,7 +63,7 @@ class DevProcess { // task.createDevMapping() // task.deleteAddr() // task.deleteAddr2() -// task.synchDevTo100() + task.synchDevTo100() // task.addNewDev2() // task.addAlias() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy index d56afba8a..cf1b49203 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy @@ -20,6 +20,10 @@ class DevUpdateTestBatch { def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; DBCollection development_zone_43_CONN = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com") + + + DBCollection development_zone_100_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL) + DBCollection DEV_MAPPING_100_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL) @@ -30,8 +34,8 @@ class DevUpdateTestBatch { static main(args) { def task = new DevUpdateTestBatch() def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx -// task.addDev(path) - task.addDevAddress(path) + task.addDev(path) +// task.addDevAddress(path) // // task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是") @@ -42,7 +46,7 @@ class DevUpdateTestBatch { def addDev(path){ ExcelUtils.modify(path,{ d-> if(!d['I'] && d['A']){ - def obj = development_zone_100_CONN.findOne(new BasicDBObject("name", d['A'])) + def obj = development_zone_43_CONN.findOne(new BasicDBObject("name", d['A'])) if(!obj){ def o = new BasicDBObject() o.put("province", d['B']) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy index 35c1fa9db..ee54817e6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy @@ -28,7 +28,7 @@ class GXDdistrictMatch { static main(args) { def task = new GXDdistrictMatch(); // task.run() - task.addOldMatch() + task.addOldMatch2() // task.runDb() // task.updateDistStatus()//恢复name_cat不正常数据 @@ -68,6 +68,32 @@ class GXDdistrictMatch { } + def addOldMatch2(){ + //老匹配 + def fileName = "D:\\数据\\建行\\房产估值\\建行业务反馈badcase.xlsx"//11城重新匹配.xlsx + ExcelUtils.multiThreadingModify(fileName, { d -> + println d['C'] + if(!d['AJ']){ + def jsonObject = OpenApiClient.addressMatch(d['C'], d['D'], "district", false,null, d['D']) + if (jsonObject != null && !jsonObject.containsKey("result")) { + d['AJ'] = jsonObject.get("id") + d['AK'] = jsonObject.get("city") + d['AL'] = jsonObject.get("district") + d['AM'] = jsonObject.get("name") + d['AN'] = jsonObject.get("address") + + def dbo = CommonDatautils.findDistrictById(jsonObject.get("id")) + if(dbo){ + d['AO'] = dbo.status + d['AP'] = dbo.name_cat + d['AQ'] = dbo.estate_type2 + } + } + } + }); + + } + def runDb(){ def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx" ExcelUtils.modify(fileName, { d -> diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy index 0fbf352f7..41d940274 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy @@ -35,7 +35,7 @@ class ThirdPriceMappingBatch extends BaseBatch{ final DBCollection dataProcessColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process", "192.168.0.235", 27018) - def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); + def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); static main(args) { runBatch(new ThirdPriceMappingBatch(), args); @@ -44,9 +44,9 @@ class ThirdPriceMappingBatch extends BaseBatch{ @Override public void process(Object args) { // //将新增数据同步过来 -// addNemCommunity() + addNemCommunity() // //接口进行匹配 - match() +// match() // addOldMatchResult() } @@ -151,8 +151,9 @@ class ThirdPriceMappingBatch extends BaseBatch{ sd.each {k,v-> udata.put(k,v) } + udata.removeField("_id") udata.put("m_date", new Date()) - thirdSeaweedMappingColl.update(query, new BasicDBObject('$set', udata)) + thirdSeaweedMappingColl.update(query, new BasicDBObject('$set', udata),false, false) }else{ sd.put("m_date", new Date()) thirdSeaweedMappingColl.save(sd) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy index 0f61101ea..9d5283a6b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy @@ -1,6 +1,7 @@ package com.fangjia.script.gj.data_handle import com.fangjia.core.RegionData +import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.core.util.GeocodeUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils @@ -900,41 +901,92 @@ class TownMiddleProcess { } def updateTownPolineBatch(){ + def i = 0 ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\乡镇街道围栏开发区数据.xlsx",{ row-> if(row['R'] && row['R']=="新增围栏"){ - def q = new BasicDBObject("city", row['B']).append("region", row['C']).append("town",row['D']) - println row['B']+" "+ row['D']+" "+TOWNPOLYLINE_COLL.count(q) - def d = TOWNPOLYLINE_COLL.findOne(q) - if(d){ - def resultMap = [:] - resultMap.type = "polygon" - def arrs = d.polyline.split("\\|") - def rets = [] - for(def s : arrs){ - def points = s.split(";") - def pointList = [] - for(def pointStr : points){ - def pointArr = pointStr.split(",") - def lon = DataHandleUtils.getDoubleRound(6, pointArr[0].toString().toDouble()) - def lat = DataHandleUtils.getDoubleRound(6, pointArr[1].toString().toDouble()) - pointList << [lon, lat] + i++ + if(i>=134){ + def q = new BasicDBObject("city", row['B']).append("region", row['C']).append("town",row['D']) + println row['B']+" "+ row['D']+" "+TOWNPOLYLINE_COLL.count(q) + def d = TOWNPOLYLINE_COLL.findOne(q) + if(d){ + def resultMap = [:] + def bd_resultMap = [:] + + resultMap.type = "polygon" + bd_resultMap.type = "polygon" + + def arrs = d.polyline.split("\\|") + def rets = [] + def bd_rets = [] + + println row['N'] +" 共有:"+arrs.size()+" 个围栏" + def index = 1 + for(def s : arrs){ + def points = s.split(";") + def pointList = [] + def bd_pointList = [] + + for(def pointStr : points){ + def pointArr = pointStr.split(",") + def lon = DataHandleUtils.getDoubleRound(6, pointArr[0].toString().toDouble()) + def lat = DataHandleUtils.getDoubleRound(6, pointArr[1].toString().toDouble()) + pointList << [lon, lat] + + // 高德转百度 + def baidu = BaiDuApiUtils.geoconv(lon, lat, 3) + def bd_lng = baidu.x + def bd_lat = baidu.y + bd_pointList << [bd_lng, bd_lat] + } + + if(index<=1){ + resultMap.coordinates = pointList + bd_resultMap.coordinates = bd_pointList + + def udata = new BasicDBObject() + udata.put("shape", resultMap) + udata.put("operator", "高娟") + udata.put("m_date", new Date()) + udata.put("bd_shape",bd_resultMap) + + def updateQuery = new BasicDBObject("name",row['N']) + DEVELOPMENT_ZONE_43_COLL.update(updateQuery, new BasicDBObject('$set',udata)) + }else{ + def dbo = DEVELOPMENT_ZONE_43_COLL.findOne(new BasicDBObject("name",row['N'])) + def obj = new BasicDBObject() + dbo.each {k,v-> + if(k!='_id'){ + obj.put(k,v) + } + } + + resultMap.coordinates = pointList + bd_resultMap.coordinates = bd_pointList + obj.put("shape", resultMap) + obj.put("operator", "高娟") + obj.put("m_date", new Date()) + obj.put("bd_shape",bd_resultMap) + obj.put("create_time", new Date()) + obj.put("block", dbo.name+"-地块"+index) + DEVELOPMENT_ZONE_43_COLL.save(obj) + } + +// rets << pointList +// bd_rets << bd_pointList + index++ } - rets << pointList + + +// if(rets.size()>1) { +// resultMap.type = "MultiPolygon" +// bd_resultMap.type = "MultiPolygon" +// } +// println d.city+" "+d.region+" "+d.town+" "+rets.size() + row['S'] = "已完成" } - resultMap.coordinates = rets - if(rets.size()>1) { - resultMap.type = "MultiPolygon" - } - println d.city+" "+d.region+" "+d.town+" "+rets.size() - - def udata = new BasicDBObject() - udata.put("shape", resultMap) - - - def updateQuery = new BasicDBObject("name",row['N']) - DEVELOPMENT_ZONE_43_COLL.update(updateQuery, new BasicDBObject('$set',udata)) - row['S'] = "已完成" } + } }) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy index c51b3f825..b5de80cae 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy @@ -53,7 +53,7 @@ class ImportIndexStats extends BaseBatch{ - public static final def dingjiYear = 2020; + public static def dingjiYear = 2020; private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" private static final EsDataSource dataSource = new EsDataSource() @@ -76,22 +76,23 @@ class ImportIndexStats extends BaseBatch{ // synch() // synchTjjIndex()//同步统计局抓取过来的指数 -// Calendar calendar=Calendar.getInstance(); -// def month= calendar.get(Calendar.MONTH); -// def year= calendar.get(Calendar.YEAR); -// def date = createDateString(year, month, 0, ""); -// calculatePriceIndex(year, month, date); + Calendar calendar=Calendar.getInstance(); + def month= calendar.get(Calendar.MONTH); + def year= calendar.get(Calendar.YEAR); + def date = createDateString(year, month, 0, ""); + calculatePriceIndex(year, month, date); // addLostIndex() - - addHistoryIndex() +// addHistoryIndex() } def addLostIndex(){ def citys = CommonDatautils.findAllCity(); + def index = 0 for(def city : citys){ + index++ if(cityList.contains(city)){ - println city +" 数据来源于统计局" + println "[${index}] "+ city +" 数据来源于统计局" continue; } def start = findStartOrEndIndex(city, 1) @@ -117,21 +118,48 @@ class ImportIndexStats extends BaseBatch{ if(nextObj && nextObj.chain){ indexs << nextObj.chain } - udata.put("chain",DataHandleUtils.getDoubleRound(1, indexs.sum()/indexs.size())) - udata.put("remark","推导") + if(indexs){ + udata.put("chain",DataHandleUtils.getDoubleRound(1, indexs.sum()/indexs.size())) + udata.put("remark","推导") + } + } + if(!dbo.yeartoyear){ + def indexs = [] + if(lastMonthObj && lastMonthObj.yeartoyear){ + indexs << lastMonthObj.yeartoyear + } + if(nextObj && nextObj.yeartoyear){ + indexs << nextObj.yeartoyear + } + if(indexs){ + udata.put("yeartoyear",DataHandleUtils.getDoubleRound(1, indexs.sum()/indexs.size())) + udata.put("remark","推导") + } + } + if(!dbo.fixedbase){ + def indexs = [] + if(lastMonthObj && lastMonthObj.fixedbase){ + indexs << lastMonthObj.fixedbase + } + if(nextObj && nextObj.fixedbase){ + indexs << nextObj.fixedbase + } + if(indexs){ + udata.put("fixedbase",DataHandleUtils.getDoubleRound(1, indexs.sum()/indexs.size())) + udata.put("remark","推导") + } + + } + if(udata){ + println city +" 补充" STATS_INDEX_COLL.update(new BasicDBObject("_id", dbo._id), new BasicDBObject('$set', udata),false, true); } -// if(!dbo.yeartoyear){ -// -// } -// if(!dbo.fixedbase){ -// -// } - } } } + + println "[${index}] "+ city +" 已处理完" } } @@ -154,7 +182,7 @@ class ImportIndexStats extends BaseBatch{ } def addHistoryIndex(){ - def s = "202101" + def s = "202001" def startDate = null; def stop = false; def runDates = [] @@ -171,7 +199,7 @@ class ImportIndexStats extends BaseBatch{ runDates << map s = DateUtils.addMonth(date, 1, "yyyyMM") - if(year==2024 && month==9){ + if(year==2024 && month==8){ stop = true; } } @@ -182,6 +210,7 @@ class ImportIndexStats extends BaseBatch{ //计算统计局没有发布的城市的价格指数 def calculatePriceIndex(def year, def month, def date){ +// def citys = ["五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]// def citys = CommonDatautils.findAllCity(); // def date = createDateString(year, month, 0, ""); def haveIndexCitys = findPriceIndexCitys(date.toInteger()); @@ -193,7 +222,7 @@ class ImportIndexStats extends BaseBatch{ def newCitys = [] for(def i=0;i=201601){ + dingjiYear = 2015 + }else if(date.toInteger()>=202101){ + dingjiYear = 2020 + }else if(date.toInteger()>=202601){ + dingjiYear = 2025 + }else if(date.toInteger()>=203101){ + dingjiYear = 2030 + } fixedThreadPool.execute(new Runnable() { void run() { try{ @@ -220,7 +258,7 @@ class ImportIndexStats extends BaseBatch{ } }); } - latch.await(20, TimeUnit.SECONDS); + latch.await(1, TimeUnit.MINUTES); } def calculating(city, year, month, date){ @@ -647,6 +685,7 @@ class ImportIndexStats extends BaseBatch{ DBObject q = new BasicDBObject(); q.put("yearmonth", date); q.put("cat", "district") + q.put("type", "城市") DBObject keys = new BasicDBObject(); keys.put("city", 1) @@ -666,6 +705,7 @@ class ImportIndexStats extends BaseBatch{ q.put("yearmonth", date); q.put("city", city); q.put("cat", "district"); + q.put("type", "城市") DBObject keys = new BasicDBObject(); keys.put("city", 1) @@ -684,6 +724,7 @@ class ImportIndexStats extends BaseBatch{ dbo.put("source",result.source); dbo.put("remark",result.remark); dbo.put("cat",result.cat); + dbo.put("type",result.type); dbo.put("c_date",new Date()); if(result.markup){ dbo.put("chain", result.markup.toDouble()); From bda9cca523525788cff9c7934024e253e4e6eec2 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 22 Oct 2024 18:24:49 +0800 Subject: [PATCH 14/35] update --- .../DealAndThirdPriceDataProcess.groovy | 89 +++++++++++++------ 1 file changed, 63 insertions(+), 26 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index 023cc157d..d2c8a1f84 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -5,9 +5,14 @@ import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject import groovy.sql.Sql import org.bson.types.ObjectId +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + /** * 建行成交数据表优化 */ @@ -24,6 +29,8 @@ class DealPriceDataProcess { def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified", "postgres", "postgres", "org.postgresql.Driver") + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + static main(args) { def task = new DealPriceDataProcess() // task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据 @@ -394,43 +401,73 @@ class DealPriceDataProcess { def rows = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def set = new HashSet() + def index = 0 for(def city : citys) { + index ++ + if(index<145){ + continue + } + println index +" "+city def cityid = cityCodeDatas.get(city) def rets = findPeGteThirdParty2(cityid, DateUtils.formatDateByPattern(new Date(),"yyyyMM"))//当前月日期 println city +" "+rets.size() if(rets){ set.add(city) } - for(def m : rets){ - -// def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}" -// sqlInstance236.executeUpdate(sql) - def querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}" - def total = 0 - pgsqlInstance.eachRow(querySql, { r -> - total = r.count - }) - if(total>0){ - println city +" "+m.commid+" (按价格)删除:"+total - def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}" - pgsqlInstance.executeUpdate(sql) - }else{ - - querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} " - pgsqlInstance.eachRow(querySql, { r -> - total = r.count - }) - if(total>0){ - println city +" "+m.commid+" (按小区)删除:"+total - def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} " - pgsqlInstance.executeUpdate(sql) - } - } - } + processDetele(city, cityid, month, rets) } println set.toString() } + + def processDetele(city, cityid, month, rets){ + + final CountDownLatch latch = new CountDownLatch(rets.size()) + + for(def i=0;i + total = r.count + }) + if(total>0){ + println city +" "+m.commid+" (按价格)删除:"+total + def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}" + pgsqlInstance.executeUpdate(sql) + }else{ + + querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} " + pgsqlInstance.eachRow(querySql, { r -> + total = r.count + }) + if(total>0){ + println city +" "+m.commid+" (按小区)删除:"+total + def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} " + pgsqlInstance.executeUpdate(sql) + } + } + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + } + }); + } + + latch.await() + } + + + + + def findThirdCount(sqlInstance236, sql){ def count = 0 sqlInstance236.eachRow(sql,{r-> From 3ab5c516a9d912d5f8db5c5c916d4d41bc02cccd Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Wed, 23 Oct 2024 11:21:01 +0800 Subject: [PATCH 15/35] =?UTF-8?q?=E7=AC=AC=E4=B8=89=E6=96=B9=E5=8C=B9?= =?UTF-8?q?=E9=85=8D=E5=B0=8F=E5=8C=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/gj/data_handle/ThirdPriceMappingBatch.groovy | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy index 41d940274..5da686848 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy @@ -46,9 +46,9 @@ class ThirdPriceMappingBatch extends BaseBatch{ // //将新增数据同步过来 addNemCommunity() // //接口进行匹配 -// match() + match() -// addOldMatchResult() + addOldMatchResult() } def match(){ @@ -119,7 +119,12 @@ class ThirdPriceMappingBatch extends BaseBatch{ def addNemCommunity(){ def citys = CommonUtils.findAllCity() def total = 0 + def index = 0 for(def c : citys){ + index ++ + if(index < 70){ + continue + } def crawlDatas = thirdPartyPriceColl.find(new BasicDBObject("city", c).append("s_date", curDate)).toList() def addCount = addDatasToDB(crawlDatas) println c + " 新增:"+ addCount From 2df46ebb2eb14029f8b605ed8ab41dad005f8557 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Wed, 23 Oct 2024 17:52:00 +0800 Subject: [PATCH 16/35] =?UTF-8?q?=E6=9F=A5=E9=AB=98=E5=BE=B7=E5=9C=B0?= =?UTF-8?q?=E5=9B=BE=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../data_handle/AddDistrictShapeBatch.groovy | 110 ++++++++++++++++-- 1 file changed, 100 insertions(+), 10 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy index 5093ecb3a..d250b8944 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDistrictShapeBatch.groovy @@ -1,14 +1,18 @@ package com.fangjia.script.gj.data_handle +import com.fangjia.core.util.HttpRequest import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.rabbitmq.client.impl.AMQImpl +import groovy.json.JsonSlurper +import net.sf.json.JSONObject import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch @@ -36,6 +40,8 @@ class AddDistrictShapeBatch extends BaseBatch{ @Override public void process(Object args) { + +// findGaoDeDatas2("苏州市","吴中区", "水岸清华瀚宫" ) def citys = CommonDatautils.findAllCity() def index = 1 for(def c : citys){ @@ -228,7 +234,7 @@ class AddDistrictShapeBatch extends BaseBatch{ def fenceMap = null for(def name : alias){ - fenceMap = findGaoDeDatas(fullCityName, fullDistName, name) + fenceMap = findGaoDeDatas2(fullCityName, fullDistName, name) if(!fenceMap){ continue } @@ -244,15 +250,14 @@ class AddDistrictShapeBatch extends BaseBatch{ o.put("data_id", fenceMap.id) o.put("comm_id", d._id.toString()) def resultMap = [:] - - def shape = fenceMap.shape - def points_x = fenceMap.shape.polygon.x - def points_y = fenceMap.shape.polygon.y - def list = [] - for(def i=0;i Date: Fri, 25 Oct 2024 18:53:18 +0800 Subject: [PATCH 17/35] =?UTF-8?q?=E6=88=90=E4=BA=A4=E6=9A=82=E6=97=B6?= =?UTF-8?q?=E5=8E=BB=E6=8E=89=E5=8C=B9=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/BigdataDealBatch.groovy | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index a57be2c48..f75cbb9f1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -71,11 +71,11 @@ class BigdataDealBatch extends BaseBatch{ def citys = CcbCommonUtils.getRunningCitys(args) println "curDate:"+curDate - def allCitys = CommonDatautils.findAllCity() - for(def city : allCitys){ - addDealPriceMatch(city); - } - updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月 +// def allCitys = CommonDatautils.findAllCity() +// for(def city : allCitys){ +// addDealPriceMatch(city); +// } +// updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月 def rows = [] From d33b66de30482a9d90ed48d1b4df95d8fe96e197 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 29 Oct 2024 09:49:35 +0800 Subject: [PATCH 18/35] update --- .../ccbevaluation/AddThirdPriceSample.groovy | 122 ++++++++-- .../fangjia/script/ccbevaluation/Test.groovy | 67 +++--- .../script/commData/CommonDatautils.groovy | 67 +++++- .../data_handle/AutoAddDistrictBatch.groovy | 2 +- .../gj/data_handle/RegionStandardBatch.groovy | 210 ++++++++++++++++++ .../gj/data_handle/TownCenterBatch.groovy | 50 ++++- .../gj/data_handle/TownMiddleProcess.groovy | 55 ++++- .../data_handle/TownshipStrandUpdate.groovy | 24 +- .../fangjia/script/gj/poc/AddPriceTest.groovy | 23 +- .../script/gj/xiamen/ElasticUtils89.groovy | 17 +- .../script/gj/xiamen/SeaweedDataSynch.groovy | 8 +- .../com/fangjia/script/util/ExcelUtils.groovy | 2 +- 12 files changed, 552 insertions(+), 95 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPriceSample.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPriceSample.groovy index a36d9d715..d3df8510d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPriceSample.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPriceSample.groovy @@ -1,11 +1,18 @@ package com.fangjia.script.ccbevaluation import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DateUtil +import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + /** * 将国信达及成交对标数据加入到第三方均价表,做为均价参考 */ @@ -17,18 +24,24 @@ class AddThirdPriceSample { public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + static main(args) { def task = new AddThirdPriceSample() - task.run() //添加第三方 fccs->保存的是国信达的参考价格 csjj->保存的是国信达的成交数据 -// task.addTrade() //添加成交 +// task.run() //添加第三方 fccs->保存的是国信达的参考价格 csjj->保存的是国信达的成交数据 + task.addTrade() //添加成交 + +// task.matchOldData() +// task.noInCcbCity() + } def run(){ - for(def i=0;i<19;i++){ - ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\国信达参考价作为第三方入库.xlsx",{d-> + for(def i=0;i<1;i++){ + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\国信达指导价录入-1024.xlsx",{d-> if(d['A'] && d['B']){ def dist = CommonDatautils.findDistrictById(d['A'].toString().trim()) if(dist){ - def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202408) + def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202409) // def udata = new BasicDBObject("csjj_esf_price", d['C'].toString().toDouble().toInteger()) def udata = new BasicDBObject("fccs_esf_price", d['B'].toString().toDouble().toInteger()) AVERAGE_STATISTICS.update(q,new BasicDBObject('$set', udata),false, false) @@ -44,15 +57,16 @@ class AddThirdPriceSample { } def addTrade(){ - def citys = ["武汉","南京","杭州","长沙","郑州","合肥","宁波","南通","嘉兴","珠海","泉州","绍兴","湖州","镇江"] - def rets = findTrades(citys) -// DEAL_PRICE_COLL.insert(rets) +// def citys = ["武汉","南京","杭州","长沙","郑州","合肥","宁波","南通","嘉兴","珠海","泉州","绍兴","湖州","镇江"] + def rets = findTrades() + DEAL_PRICE_COLL.insert(rets) } - def findTrades(citys){ + def findTrades(){ def q = new BasicDBObject() - q.put("city", new BasicDBObject('$in', citys)) - q.put("flag", 1) + q.put("ccb_price_remark","建行无此城市成交") +// q.put("city", new BasicDBObject('$in', citys)) +// q.put("flag", 1) def dbcursor = NO_PRICE_DEAL_PRICE_COLL.find(q) def rets = [] def index = 0 @@ -62,7 +76,7 @@ class AddThirdPriceSample { udata.put("_id",db._id) udata.put("url",db.url) - udata.put("area",db.area) + udata.put("area",db.area.toString().toDouble()) udata.put("avg_price",db.avg_price.toString().toDouble().toInteger()) udata.put("build_structure",db.build_structure) udata.put("build_type",db.build_type) @@ -73,8 +87,14 @@ class AddThirdPriceSample { udata.put("elevator",db.elevator) udata.put("fitment",db.fitment) udata.put("floor_info",db.floor_info) - udata.put("hall",db.hall.toString().toInteger()) - udata.put("height",db.height.toString().toInteger()) + if(db.hall){ + udata.put("hall",db.hall.toString().toInteger()) + } + + if(db.height){ + udata.put("height",db.height.toString().toInteger()) + } + udata.put("housing_age",db.housing_age) udata.put("lift",db.lift.toString().toInteger()) udata.put("lift_household",db.lift_household.toString().toInteger()) @@ -83,7 +103,7 @@ class AddThirdPriceSample { udata.put("room", db.room.toString().toInteger()) udata.put("room_structure",db.room_structure) udata.put("source",db.source) - udata.put("total_price",db.total_price.toString().toInteger()) + udata.put("total_price",db.total_price.toString().toDouble().toInteger()) udata.put("trade_date",db.trade_date) udata.put("trade_property",db.trade_property) udata.put("house_using",db.house_using) @@ -98,11 +118,79 @@ class AddThirdPriceSample { udata.put("match_id",db.match_id) udata.put("price_flag", "国信达成交-推导") - DEAL_PRICE_COLL.save(udata) +// DEAL_PRICE_COLL.save(udata) println index -// rets << udata + rets << udata } return rets } + def matchOldData(){ + def q = new BasicDBObject("source","贝壳找房").append("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2024-09-01","yyyy-MM-dd"))) + def list = DEAL_PRICE_COLL.find(q).toList() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(list.size()) + for(def i=0;i -// def dist = CommonDatautils.findDistrictById(d['A']) -// if(dist){ -// d['G'] = dist.city -// d['H'] = dist.region -// d['I'] = dist.name -// } -//}) + @@ -1331,34 +1323,45 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ -def fileName = "D:\\数据\\建行\\房产估值\\6个城市待优化_9月xlsx.xlsx" -// -ExcelUtils.multiThreadingModify(fileName, {d-> - if(d['G']){ -// def region = RegionData.formatRegion(d['A'], d['J']) -// if(region){ -// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district") +//def fileName = "D:\\数据\\建行\\房产估值\\6个城市待优化_9月xlsx.xlsx" +//// +//ExcelUtils.multiThreadingModify(fileName, {d-> +// if(d['G']){ +//// def region = RegionData.formatRegion(d['A'], d['J']) +//// if(region){ +//// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district") +//// if(dist){ +//// d['B'] = dist.region +//// d['C'] = dist.name +//// d['L'] = dist.status-dist.name_cat +//// } +//// } +// if(d['G'] instanceof String){ +// def dist = CommonDatautils.findDistrictById(d['G']) // if(dist){ -// d['B'] = dist.region -// d['C'] = dist.name -// d['L'] = dist.status-dist.name_cat +//// d['A'] = dist.city +//// d['B'] = dist.region +//// d['C'] = dist.name +//// d['K'] = dist.estate_type2 +//// d['L'] = dist.status +//// d['M'] = dist.name_cat +// +// d['H'] = dist.name // } // } - if(d['G'] instanceof String){ - def dist = CommonDatautils.findDistrictById(d['G']) - if(dist){ -// d['A'] = dist.city -// d['B'] = dist.region -// d['C'] = dist.name -// d['K'] = dist.estate_type2 -// d['L'] = dist.status -// d['M'] = dist.name_cat +// } +//},0) - d['H'] = dist.name - } - } + +def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20241025人工指导价.xlsx" +ExcelUtils.multiThreadingModify(fileName, { d -> + def dist = CommonDatautils.findDistrictById(d['J']) + if(dist){ + d['A'] = dist.city + d['B'] = dist.region + d['C'] = dist.name } -},0) +}) //def file = "D:\\数据\\建行\\房产估值\\指导价\\5城价格翻新6-21.xlsx" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 5e2937768..cc8d27021 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -4785,7 +4785,6 @@ class CommonDatautils { def q = new BasicDBObject() q.put("type","区域") q.put("district_diff",1) - q.put("city",new BasicDBObject('$in', ["深圳市"])) def keys = new BasicDBObject() keys.put("province",1) @@ -4814,6 +4813,72 @@ class CommonDatautils { return regionMap } + + static def queryAllRegionNameByCity(){ + + def q = new BasicDBObject() + q.put("type","区域") + q.put("district_diff",1) + + def keys = new BasicDBObject() + keys.put("city_short",1) + keys.put("district_short",1) + keys.put("alias",1) + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def retMap = [:] + for(def d : dbcursor){ + def regionMap = retMap.get(d.city_short) + if(!regionMap){ + regionMap = [:] + retMap.put(d.city_short, regionMap) + } + regionMap.put(d.district_short, d.alias) + } + + return retMap + } + + static def queryAllRegionFullNameByCity(){ + def q = new BasicDBObject() + q.put("type","区域") + q.put("district_diff",1) + + def keys = new BasicDBObject() + keys.put("city",1) + keys.put("district",1) + keys.put("alias",1) + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def retMap = [:] + for(def d : dbcursor){ + def regionMap = retMap.get(d.city) + if(!regionMap){ + regionMap = [:] + retMap.put(d.city, regionMap) + } + regionMap.put(d.district, d.alias) + } + + return retMap + } + + static def queryAllStreetNameByCity(city){ + + def q = new BasicDBObject() + q.put("type","乡镇街道") + q.put("city", city) + + def keys = new BasicDBObject() + keys.put("street",1) + keys.put("alias",1) + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def retMap = [:] + for(def d : dbcursor){ + retMap.put(d.street, d.alias) + } + + return retMap + } + static def queryCommunityByCity(){ def q = new BasicDBObject() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy index f27ee361f..f6a8c1208 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy @@ -49,7 +49,7 @@ class AutoAddDistrictBatch extends BaseBatch{ def fileName = "/home/goojia/gj/add_dist_info.xlsx" if (CommonDatautils.isWindow()) { - fileName = "D:\\数据\\数据建设\\名录集\\第三方待入库数据.xlsx" + fileName = "D:\\数据\\数据建设\\小区\\第三方待入库数据.xlsx" } ExcelUtils.write(fileName, list, "待入库数据") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy new file mode 100644 index 000000000..3bc2f080a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy @@ -0,0 +1,210 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.core.RegionData +import com.fangjia.script.BaseBatch +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.xiamen.ElasticUtils89 +import com.fangjia.script.util.DbUtils +import com.mongodb.BasicDBObject +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +/** + * 区域标准化 + * 1.小区库区域标准化,包含小区,商铺,写字楼 + * 2.poi数据库中 区域标准化 + */ +class RegionStandardBatch extends BaseBatch{ + + def fullCityMap = CommonDatautils.queryFullNameCity() + def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity() + def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity() + + static main(args) { + runBatch(new RegionStandardBatch(), args) + } + + @Override + void process(Object args) { +// seaweedRegionStandard() + poiRegionStandard() + } + + def seaweedRegionStandard(){ + seaweedRegionStandardByCat("shop") + } + def poiRegionStandard(){ + def citys = ["三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德"]//CommonDatautils.findAllCity() + for(def city : citys){ + + def data = fullCityMap.get(city) + def fullCityName = null + if(data){ + fullCityName = data.city + } + if(fullCityName){ + + def rMap = regionFullNameMap.get(fullCityName) + if(rMap){ + def set = rMap.keySet() + def rets = findEsPoiByCity(fullCityName) + println city +" "+rets.size() + if(set.size()>1){ + poiHaveRegionProcess(city, set, rMap, rets) + }else{ +// haveStreetProcess(city, set, cat) + } + } + }else{ + println "没找到相应的全称:"+city + } + + } + + } + + + def seaweedRegionStandardByCat(cat){ + def citys = CommonDatautils.findAllCity() + for(def city : citys){ + + def rMap = regionMap.get(city) + if(rMap){ + def set = rMap.keySet() + if(set.size()>1){ + haveRegionProcess(city, set, rMap, cat) + }else{ + haveStreetProcess(city, set, cat) + } + } + } + + } + + def haveRegionProcess(city, set, rMap, cat){ + def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name"]) + for(def d : districts){ + if(set.contains(d.region)){ + continue + } + def updateRegion = null + for(def key : set){ + def alias = rMap.get(key) + if(alias.contains(d.region)){ + updateRegion = key + break; + } + } + if(updateRegion){ + def udata = new BasicDBObject() + udata.put("region", updateRegion) + udata.put("m_date", new Date()) + println d.city +" "+d.region+" "+d.name +" 区域修改为:"+updateRegion + DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + }else{ + def udata = new BasicDBObject() + udata.put("status", 1) + udata.put("m_date", new Date()) + println d.city +" "+d.region+" "+d.name +" 状态修改为:删除" + DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + } + } + } + + def haveStreetProcess(city, set, cat){ + def data = fullCityMap.get(city) + def fullCityName = null + if(data){ + fullCityName = data.city + } + def streetMap = CommonDatautils.queryAllStreetNameByCity(fullCityName) + if(streetMap){ + set = streetMap.keySet() + def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name"]) + for(def d : districts){ + if(set.contains(d.region)){ + continue + } + + def updateRegion = null + def flag = null + for(def key : set){ + def alias = streetMap.get(key) + if((alias && alias.contains(d.region)) || (!alias && key.contains(d.region))){ + def newRegion = RegionData.formatRegion(d.city, key) + if(newRegion!=d.region){ + updateRegion = newRegion + } + flag = "pass" + break; + } + } + if(updateRegion){ + def udata = new BasicDBObject() + udata.put("region", updateRegion) + udata.put("m_date", new Date()) + println d.city +" "+d.region+" "+d.name +" 区域修改为:"+updateRegion +// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + }else{ + if(flag!="pass"){ + def udata = new BasicDBObject() + udata.put("status", 1) + udata.put("m_date", new Date()) + println d.city +" "+d.region+" "+d.name +" 状态修改为:删除" +// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + } + } + } + } + } + + def findEsPoiByCity(city){ + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) + def rets = [] + + def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi_shards", queryBuilder) + for(def d: datas){ + def o = new BasicDBObject() + o.put("city", d.city) + o.put("district", d.district) + o.put("name", d.name) + o.put("township", d.township) + o.put("location", d.location) + rets << o + }) + return rets + } + + + def poiHaveRegionProcess(city, set, rMap, datas){ + for(def d : datas){ + if(set.contains(d.region)){ + continue + } + def updateRegion = null + for(def key : set){ + def alias = rMap.get(key) + if(alias.contains(d.region)){ + updateRegion = key + break; + } + } + if(updateRegion){ + def udata = new BasicDBObject() + udata.put("region", updateRegion) + udata.put("m_date", new Date()) + println d.city +" "+d.region+" "+d.name +" 区域修改为:"+updateRegion + + }else{ + def udata = new BasicDBObject() + udata.put("status", 1) + udata.put("m_date", new Date()) + println d.city +" "+d.region+" "+d.name +" 状态修改为:删除" + + } + } + } + + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownCenterBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownCenterBatch.groovy index e1bfa8969..b776efc53 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownCenterBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownCenterBatch.groovy @@ -22,53 +22,85 @@ class TownCenterBatch extends BaseBatch{ @Override void process(Object args) { - def shapes = findHasShapeTown() - def centers = findAllCenters() - checkCenter(shapes, centers) + def city = "上海市" + def shapes = findHasShapeTown(city) + def centers = findAllCenters(city) + def retMap = checkCenter(shapes, centers) + retMap.each {key,values-> + if(values.size()>1){ + println values + } + } } def checkCenter(shapes, centers){ def map = [:] + def index = 0 for(def town : shapes){ + index ++ def points = InsidePolygon.createPoint4(town.shape.coordinates) //1.先判断当前镇的中心点是否在围栏中 if(town.location){ - def flag = InsidePolygon.isInArea(town.location.lng, town.location.lat, points) + def flag = InsidePolygon.isInArea(town.location.lon, town.location.lat, points) if(!flag){ println town.city + " "+town.district+" "+town.street+" 不在本身围栏中" + }else{ + def list = map.get(town.city + "#"+town.district+"#"+town.street) + if(!list){ + list = [] + map.put(town.city + "#"+town.district+"#"+town.street, list) + } + list << town.city + "#"+town.district+"#"+town.street } } //2.判断是否有其它点在围栏中 for(def center : centers){ if(center._id.toString() != town._id.toString()){ - def flag = InsidePolygon.isInArea(center.location.lng, center.location.lat, points) + def flag = InsidePolygon.isInArea(center.location.lon, center.location.lat, points) if(flag){ def list = map.get(town.city + "#"+town.district+"#"+town.street) if(!list){ list = [] - map.put(town.city + "#"+town.district+"#"+town.street) + map.put(town.city + "#"+town.district+"#"+town.street, list) } list << center.city + "#"+center.district+"#"+center.street println center.city + " "+center.district+" "+center.street+" 在 "+town.city + " "+town.district+" "+town.street+" 围栏中" } } } + def list = map.get(town.city + "#"+town.district+"#"+town.street) + if(list){ + println index+" "+town.city + "#"+town.district+"#"+town.street + " "+list.size() + } } return map } - def findHasShapeTown(){ + def findHasShapeTown(city){ def q = new BasicDBObject() q.put("type", "乡镇街道") q.put("shape", new BasicDBObject('$exists', true)) - return ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(2000).toList() + if(city){ + q.put("city", city) + } + + def keys = new BasicDBObject() + keys.put("location", 1) + keys.put("city", 1) + keys.put("district", 1) + keys.put("street", 1) + keys.put("shape", 1) + return ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).toList() } - def findAllCenters(){ + def findAllCenters(city){ def q = new BasicDBObject() q.put("type", "乡镇街道") q.put("location", new BasicDBObject('$exists', true)) + if(city){ + q.put("city", city) + } def keys = new BasicDBObject() keys.put("location", 1) keys.put("city", 1) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy index 9d5283a6b..4ecd49ed0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy @@ -78,7 +78,7 @@ class TownMiddleProcess { //对比中间表的街道与community街道是否一致,因为元极反馈类似“高密市醴泉街道”解析有问题,要去掉“高密市”,然后去了中间表 // updateCommunityDiffStreet() // updateTownPoline()//更新乡镇街道围栏 - updateTownPolineBatch()//批量更新乡镇街道围栏 +// updateTownPolineBatch()//批量更新乡镇街道围栏 // checkTown() //区域民政部名称更新到乡镇街道中 // updateTownshipByRegion() @@ -94,6 +94,8 @@ class TownMiddleProcess { // addTownCenter()//百度接口添加中心点 // checkCenterInTownRang()//检测乡镇街道中心点是否在围栏中 + + addSpecialCityTownAlias() } @@ -1301,6 +1303,57 @@ class TownMiddleProcess { for(def d : dbcursor){ } + } + def addSpecialCityTownAlias(){ + def fullCityMap = CommonDatautils.queryFullNameCity() + + def citys = ["东莞","中山","嘉峪关","昆玉","石河子","阿拉尔","图木舒克","五家渠","济源","仙桃","潜江","天门","神农架","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中","北屯","铁门关","双河","可克达拉"] + def regionMap = CommonDatautils.getAllCityFullRegions() + for(def city : citys){ + def townMap = regionMap.get(city) + def streets = findAllStreetByCity(city) + for(def s : streets){ + def set = townMap.keySet() + def alias = null + for(def name : set){ + def townObj = townMap.get(name) + if(townObj && townObj.pattern && townObj.pattern.contains(s.street)){ + alias = townObj.pattern + break; + } + } + if(alias){ + def newAlias = [] + for(def n : alias){ + if(n!=s.street){ + newAlias << n + } + } + if(alias){ + def udata = new BasicDBObject() + udata.put("alias", newAlias) + ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id", s._id), new BasicDBObject('$set', udata),false, false) + } + } + } + } + } + + + def findAllStreetByCity(city){ + def data = fullCityMap.get(city) + def fullCityName = null + if(data){ + fullCityName = data.city + } + + def q = new BasicDBObject() + q.put("type", "乡镇街道") + q.put("city", fullCityName) + + def keys = new BasicDBObject() + keys.put("street", 1) + return ADMINISTRATIVE_MAPPING_COLL.find(q, keys).toList() } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy index 2a6c1d7a3..d99e249c9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy @@ -42,7 +42,7 @@ class TownshipStrandUpdate { // task.updateTown() // task.stat() -// task.updateDiffRegionTown() + task.updateDiffRegionTown() // task.cleanTown() // task.checkTownFromMongo() // task.updateTownMapping() @@ -423,29 +423,29 @@ class TownshipStrandUpdate { } def updateDiffRegionTown(){ - def fileName = "D:\\数据\\数据建设\\乡镇街道\\shop-不同区域.xlsx" - ExcelUtils.multiThreadingModify(fileName,{d-> - def lists = CommonDatautils.findDistrictList(d['B'], null, d['D'], "shop") + def fileName = "D:\\数据\\数据建设\\乡镇街道\\district-不同区域.xlsx" + ExcelUtils.modify(fileName,{d-> + def lists = CommonDatautils.findDistrictList(d['A'], null, d['C'], "district") d['AA'] = lists.size() //只处理库中只有一个的 - if(lists && lists.size()==1 && d['Z']=="包含" && !d['AB']){ - def newRegion = RegionData.formatRegion(d['B'], d['M']) + if(lists && lists.size()==1 && d['R']){ + def newRegion = RegionData.formatRegion(d['A'], d['R']) if(newRegion){ - if(d['X'] && d['X'].toString().contains(d['M']+" "+d['N'])){ //地图街道在中间表中 +// if(d['X'] && d['X'].toString().contains(d['M']+" "+d['N'])){ //地图街道在中间表中 def dbo = lists[0] - def arrs = d['X'].split(" ") +// def arrs = d['X'].split(" ") def udata = new BasicDBObject() udata.put("region", newRegion) udata.put("m_date", new Date()) - udata.put("township", arrs[1]) +// udata.put("township", arrs[1]) DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata)) - CommonDatautils.saveOptLog(dbo.city, dbo.region, dbo.name, "shop", "update","高娟") + CommonDatautils.saveOptLog(dbo.city, dbo.region, dbo.name, "district", "update-region","高娟") d['AB'] = "已处理" - } +// } } } - }) + },1) } def cleanTown(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy index eed8d9fda..d314427a9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy @@ -2,6 +2,7 @@ package com.fangjia.script.gj.poc import com.fangjia.script.ccbevaluation.CcbCommonUtils import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DateUtil import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils @@ -37,9 +38,9 @@ class AddPriceTest { task.run(); } def run(){ - def filePath = "D:\\数据\\建行\\房产估值\\建行业务反馈badcase.xlsx" -// addGXDPrice(filePath) - addCcbPrice(filePath) + def filePath = "D:\\数据\\建行\\房产估值\\202410测试偏差较大案例统计数据test.xlsx" + addGXDPrice(filePath) +// addCcbPrice(filePath) // addThirdPrice(filePath) // addFjwPrice(filePath) // exportPkPrice() @@ -269,12 +270,16 @@ class AddPriceTest { def addGXDPrice(filepath){ ExcelUtils.multiThreadingModify(filepath, { d -> - if (!d['M']) { - def dist = CommonDatautils.findDistrictById(d['B']) + if (!d['B']) { + def dist = CommonDatautils.findDistrictById(d['A']) if(dist){ def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name) // def gxdPrice = findGXDPrice(d['D'], d['E'], d['C']) - d['M'] = gxdPrice + if(gxdPrice){ + d['B'] = gxdPrice.price + d['C'] = DateUtils.formatDateByPattern(gxdPrice.c_date, "yyyy-MM-dd") + } + } } }) @@ -288,11 +293,11 @@ class AddPriceTest { q.put("district_name", name) q.put("source", "GXD") def dbcursor = confirm_price_date.find(q).sort(new BasicDBObject("c_date", -1)).limit(1) - def price = null + def obj = null for(def d : dbcursor){ - price = d.price + obj = d } - return price + return obj } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy index 2d2f76f24..db5362ca7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy @@ -236,6 +236,7 @@ class ElasticUtils89 { searchRequest.scroll(scroll); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query(qb); + searchSourceBuilder.size(100) searchRequest.source(searchSourceBuilder); SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); String scrollId = searchResponse.getScrollId(); @@ -247,14 +248,14 @@ class ElasticUtils89 { searchHit.getSourceAsMap().put("id", searchHit.getId()); results << searchHit.getSourceAsMap() } - if(results.size()%100000==0){ - println "."+ new Date() - } - SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); - scrollRequest.scroll(scroll); - searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT); - scrollId = searchResponse.getScrollId(); - searchHits = searchResponse.getHits().getHits(); +// if(results.size()%100000==0){ +// println "."+ new Date() +// } +// SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); +// scrollRequest.scroll(scroll); +// searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT); +// scrollId = searchResponse.getScrollId(); +// searchHits = searchResponse.getHits().getHits(); } ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); clearScrollRequest.addScrollId(scrollId); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy index 5b5e79b62..e78bc7018 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy @@ -150,8 +150,8 @@ class SeaweedDataSynch extends BaseBatch{ final CountDownLatch latch = new CountDownLatch(dists.size()) for(def i=0;i Date: Tue, 29 Oct 2024 15:59:46 +0800 Subject: [PATCH 19/35] update --- .../gj/data_handle/RegionStandardBatch.groovy | 84 ++++++++++++++++--- .../script/gj/xiamen/ElasticUtils89.groovy | 6 +- 2 files changed, 75 insertions(+), 15 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy index 3bc2f080a..aeac02b63 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy @@ -6,6 +6,10 @@ import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.xiamen.ElasticUtils89 import com.fangjia.script.util.DbUtils import com.mongodb.BasicDBObject +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.update.UpdateRequest +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.common.xcontent.XContentType import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders @@ -162,7 +166,7 @@ class RegionStandardBatch extends BaseBatch{ QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) def rets = [] - def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi_shards", queryBuilder) + def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi", queryBuilder) for(def d: datas){ def o = new BasicDBObject() o.put("city", d.city) @@ -170,39 +174,95 @@ class RegionStandardBatch extends BaseBatch{ o.put("name", d.name) o.put("township", d.township) o.put("location", d.location) + o.put("status", d.status) + o.put("id", d.id) rets << o - }) + } return rets } def poiHaveRegionProcess(city, set, rMap, datas){ + def results = [] for(def d : datas){ - if(set.contains(d.region)){ + if(set.contains(d.district)){ continue } def updateRegion = null for(def key : set){ def alias = rMap.get(key) - if(alias.contains(d.region)){ + if(alias.contains(d.district)){ updateRegion = key break; } } if(updateRegion){ - def udata = new BasicDBObject() - udata.put("region", updateRegion) - udata.put("m_date", new Date()) - println d.city +" "+d.region+" "+d.name +" 区域修改为:"+updateRegion + def m = [:] + m.put(d.id, ["district":updateRegion]) + results << m + if(results.size()%1==0){ + updateRegionByEs(results) + results.clear() + } + println d.city +" "+d.district+" "+d.name +" "+d.status+" 区域修改为:"+updateRegion }else{ - def udata = new BasicDBObject() - udata.put("status", 1) - udata.put("m_date", new Date()) - println d.city +" "+d.region+" "+d.name +" 状态修改为:删除" + def m = [:] + m.put(d.id, ["status":1]) + results << m + if(results.size()%1==0){ + updateRegionByEs(results) + results.clear() + } + + println d.city +" "+d.district+" "+d.name +" "+d.status+" 状态修改为:删除" } } + + if(results.size()%1>0){ + updateRegionByEs(results) + results.clear() + } + } + + + + def updateRegionByEs(rows){ + def client = ElasticUtils89.getClient("192.168.0.89") + def bulkRequest = new BulkRequest() + + rows.each { row -> + def id = row.keySet().iterator().next() + def doc = row.get(id) + + def updateRequest = new UpdateRequest("didata_map_poi", id) + updateRequest.doc(doc, XContentType.JSON) + bulkRequest.add(updateRequest) + } + + try { + // 执行批量更新请求 + def bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT) + + // 检查响应 + if (bulkResponse.hasFailures()) { + println("批量更新中有失败的操作:") + bulkResponse.buildFailureMessage().each { println(it) } + } else { + println("批量更新成功!") + } + } catch (Exception e) { + e.printStackTrace() + } finally { + // 关闭客户端 + try { + client.close() + } catch (Exception e) { + e.printStackTrace() + } + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy index db5362ca7..badba650a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy @@ -236,13 +236,13 @@ class ElasticUtils89 { searchRequest.scroll(scroll); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query(qb); - searchSourceBuilder.size(100) +// searchSourceBuilder.size(1000) searchRequest.source(searchSourceBuilder); SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); String scrollId = searchResponse.getScrollId(); SearchHit[] searchHits = searchResponse.getHits().getHits(); def results = [] - while (searchHits != null && searchHits.length > 0) { +// while (searchHits != null && searchHits.length > 0) { for (int k=0; k < searchHits.size(); k++) { SearchHit searchHit = searchHits[k] searchHit.getSourceAsMap().put("id", searchHit.getId()); @@ -256,7 +256,7 @@ class ElasticUtils89 { // searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT); // scrollId = searchResponse.getScrollId(); // searchHits = searchResponse.getHits().getHits(); - } +// } ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); clearScrollRequest.addScrollId(scrollId); client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); From 47f09c7f8a68c3eea82f1ad6f55e51476c7785e7 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 29 Oct 2024 17:03:06 +0800 Subject: [PATCH 20/35] update --- .../gj/data_handle/RegionStandardBatch.groovy | 9 +++++---- .../script/gj/xiamen/ElasticUtils89.groovy | 20 +++++++++---------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy index aeac02b63..8533557df 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy @@ -38,7 +38,7 @@ class RegionStandardBatch extends BaseBatch{ seaweedRegionStandardByCat("shop") } def poiRegionStandard(){ - def citys = ["三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德"]//CommonDatautils.findAllCity() + def citys = ["三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德","重庆"]//CommonDatautils.findAllCity() for(def city : citys){ def data = fullCityMap.get(city) @@ -200,7 +200,7 @@ class RegionStandardBatch extends BaseBatch{ def m = [:] m.put(d.id, ["district":updateRegion]) results << m - if(results.size()%1==0){ + if(results.size()%100==0){ updateRegionByEs(results) results.clear() } @@ -210,7 +210,7 @@ class RegionStandardBatch extends BaseBatch{ def m = [:] m.put(d.id, ["status":1]) results << m - if(results.size()%1==0){ + if(results.size()%100==0){ updateRegionByEs(results) results.clear() } @@ -220,10 +220,11 @@ class RegionStandardBatch extends BaseBatch{ } } - if(results.size()%1>0){ + if(results.size()>0){ updateRegionByEs(results) results.clear() } + println city +" 处理完成" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy index badba650a..e31ded3ec 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy @@ -242,21 +242,21 @@ class ElasticUtils89 { String scrollId = searchResponse.getScrollId(); SearchHit[] searchHits = searchResponse.getHits().getHits(); def results = [] -// while (searchHits != null && searchHits.length > 0) { + while (searchHits != null && searchHits.length > 0) { for (int k=0; k < searchHits.size(); k++) { SearchHit searchHit = searchHits[k] searchHit.getSourceAsMap().put("id", searchHit.getId()); results << searchHit.getSourceAsMap() } -// if(results.size()%100000==0){ -// println "."+ new Date() -// } -// SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); -// scrollRequest.scroll(scroll); -// searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT); -// scrollId = searchResponse.getScrollId(); -// searchHits = searchResponse.getHits().getHits(); -// } + if(results.size()%100000==0){ + println "."+ new Date() + } + SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); + scrollRequest.scroll(scroll); + searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT); + scrollId = searchResponse.getScrollId(); + searchHits = searchResponse.getHits().getHits(); + } ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); clearScrollRequest.addScrollId(scrollId); client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); From 0df9701fd97c2e2d80fb85c0769e3a8a17466583 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Wed, 30 Oct 2024 09:20:45 +0800 Subject: [PATCH 21/35] update --- .../fangjia/script/gj/data_handle/RegionStandardBatch.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy index 8533557df..2fa4dbabb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy @@ -38,7 +38,8 @@ class RegionStandardBatch extends BaseBatch{ seaweedRegionStandardByCat("shop") } def poiRegionStandard(){ - def citys = ["三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德","重庆"]//CommonDatautils.findAllCity() + //"三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德", + def citys = ["重庆"]//CommonDatautils.findAllCity() for(def city : citys){ def data = fullCityMap.get(city) From 2be600c1400147de86032d524d6b21483fe8e69f Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 10 Nov 2024 10:29:53 +0800 Subject: [PATCH 22/35] =?UTF-8?q?=E6=94=B9=E4=B8=BA=E6=AF=8F=E6=97=A5?= =?UTF-8?q?=E8=BF=90=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/xiamen/RelatedTaiWanBatch.groovy | 35 +++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/RelatedTaiWanBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/RelatedTaiWanBatch.groovy index e0324502c..4efd79e7f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/RelatedTaiWanBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/RelatedTaiWanBatch.groovy @@ -12,15 +12,13 @@ import com.fangjia.script.util.MongoUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection -import com.rabbitmq.client.AMQP - /** * 厦门涉台园区数据同步 */ @BatchOwner(Watcher.GaoJuan) class RelatedTaiWanBatch extends BaseBatch{ - def curDate = Integer.parseInt(DateUtils.formatDateByPattern(new Date(), "yyyyMM")); + def curDate = Integer.parseInt(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")); DBCollection yuanqu_qiye_coll = MongoUtils.getCollectionByName("crawl", "qzd_shetai_company_detail", "192.168.0.206", 27020) def relateTaiwn_yuanqu_datas = ["福州台商投资区","杏林台商投资区","集美台商投资区","海沧台商投资区","漳州台商投资区","泉州台商投资区","漳浦台湾农民创业园","漳平台湾农民创业园","仙游台湾农民创业园","清流台湾农民创业园","福清台湾农民创业园","惠安台湾农民创业园","海峡两岸集成电路产业合作试验区(厦门、泉州产业园)","海峡两岸生技和医疗健康产业合作区","海峡两岸农业合作试验区","海峡两岸(三明)现代林业合作实验区","漳州古雷石化基地","永泰闽台农业融合发展产业园","上杭闽台农业融合发展产业园","同安闽台农业融合发展产业园","南靖闽台农业融合发展产业园","漳浦闽台农业融合发展产业园","永春闽台农业融合发展产业园","永安闽台农业融合发展产业园","邵武闽台农业融合发展产业园","平潭闽台农业融合发展产业园","南靖闽台精密机械产业园"] @@ -36,9 +34,17 @@ class RelatedTaiWanBatch extends BaseBatch{ @Override public void process(Object args) { -// createDatas() //企业库获取涉台园区企业 - updateRangeByPoint() //更改企业所在自贸区 - export() //导出数据 + + def day = new Date().getDate() + if(day == 10){ + //createDatas() //企业库获取涉台园区企业 + updateRangeByPoint() //更改企业所在自贸区 + export() //导出数据 + }else{ + exportEmptyFile() + } + + } @@ -158,4 +164,21 @@ class RelatedTaiWanBatch extends BaseBatch{ FileUtils.batchCreateFileByCustomSuffix([rdy_datas], fileName2, null, "utf-8", false, "","|+|"); XmBankFtpUtils.ftpUpload(fileName2, curDate) } + + def exportEmptyFile(){ + def fileName = "/home/goojia/xmbank/free_trade_zone_${curDate}_init.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\厦门\\涉台园区\\free_trade_zone_${curDate}_init.dat" + } + FileUtils.batchCreateFileByCustomSuffix([], fileName, null, "utf-8", false, "|@|","|+|"); + XmBankFtpUtils.ftpUpload(fileName, curDate) + + def fileName2 = "/home/goojia/xmbank/free_trade_zone_${curDate}_init.dat.rdy" + if(CommonDatautils.isWindow()){ + fileName2 = "D:\\数据\\厦门\\涉台园区\\free_trade_zone_${curDate}_init.dat.rdy" + } + def rdy_datas = ["0"] + FileUtils.batchCreateFileByCustomSuffix([rdy_datas], fileName2, null, "utf-8", false, "","|+|"); + XmBankFtpUtils.ftpUpload(fileName2, curDate) + } } From 8deebf69c5d938b7a000478ea2c593d943c2417a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 10 Nov 2024 10:32:24 +0800 Subject: [PATCH 23/35] updadte --- .../gj/data_handle/RegionStandardBatch.groovy | 76 ++++++++++++++++++- 1 file changed, 72 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy index aeac02b63..fc55cbd77 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy @@ -12,7 +12,6 @@ import org.elasticsearch.client.RequestOptions import org.elasticsearch.common.xcontent.XContentType import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders - /** * 区域标准化 * 1.小区库区域标准化,包含小区,商铺,写字楼 @@ -163,7 +162,7 @@ class RegionStandardBatch extends BaseBatch{ } def findEsPoiByCity(city){ - QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("district", "蕉岭县")) def rets = [] def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi", queryBuilder) @@ -200,7 +199,7 @@ class RegionStandardBatch extends BaseBatch{ def m = [:] m.put(d.id, ["district":updateRegion]) results << m - if(results.size()%1==0){ + if(results.size()%100==0){ updateRegionByEs(results) results.clear() } @@ -220,7 +219,7 @@ class RegionStandardBatch extends BaseBatch{ } } - if(results.size()%1>0){ + if(results.size()>0){ updateRegionByEs(results) results.clear() } @@ -265,6 +264,75 @@ class RegionStandardBatch extends BaseBatch{ } +// def queryPoiDatasByCity(){ +// def esUrl = "http://localhost:9200/your_index/_search" +// def pageSize = 1000 // 每页返回的记录数 +// def scrollSize = '1m' // scroll上下文的生存时间 +// +//// 初始化HttpClient和ObjectMapper +// CloseableHttpClient httpClient = HttpClients.createDefault() +// ObjectMapper objectMapper = new ObjectMapper() +// +//// 初始化scroll请求 +// def scrollRequest = """ +//{ +// "size": $pageSize, +// "query": { +// "match_all": {} +// }, +// "scroll": "$scrollSize" +//} +//""" +// +//// 发送scroll初始化请求 +// HttpGet scrollInitRequest = new HttpGet(esUrl) +// scrollInitRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequest, "UTF-8")) +// scrollInitRequest.setHeader("Content-Type", "application/json") +// +// HttpResponse scrollInitResponse = httpClient.execute(scrollInitRequest) +// HttpEntity entity = scrollInitResponse.getEntity() +// String scrollId = objectMapper.readValue(EntityUtils.toString(entity), Map.class).get("_scroll_id").toString() +// +//// 存储所有结果 +// def allResults = [] +// +//// 发送scroll请求获取数据,直到没有更多数据为止 +// while (true) { +// def scrollRequestWithId = """ +// { +// "scroll_id": "$scrollId", +// "scroll": "$scrollSize" +// } +// """ +// +// HttpGet scrollGetRequest = new HttpGet(esUrl + "?scroll") +// scrollGetRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequestWithId, "UTF-8")) +// scrollGetRequest.setHeader("Content-Type", "application/json") +// +// HttpResponse scrollGetResponse = httpClient.execute(scrollGetRequest) +// entity = scrollGetResponse.getEntity() +// def responseMap = objectMapper.readValue(EntityUtils.toString(entity), Map.class) +// +// def hits = responseMap.get("hits").get("hits") +// if (hits.isEmpty()) { +// break // 没有更多数据,跳出循环 +// } +// +// // 合并当前页的数据 +// allResults.addAll(hits.map({ it.get("_source") })) +// +// // 更新scrollId以进行下一次请求 +// scrollId = responseMap.get("_scroll_id").toString() +// } +// +// // 关闭HttpClient +// httpClient.close() +// +// // 输出或处理全量数据 +// println(allResults) +// +// // 注意:实际应用中,你可能需要在获取完数据后清除scroll上下文,以释放资源。 +// } } From b3a1114cc4e3cd11dd102597938a329530d31c11 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 1 Nov 2024 11:01:55 +0800 Subject: [PATCH 24/35] =?UTF-8?q?=E4=B9=A1=E9=95=87=E8=A1=97=E9=81=93?= =?UTF-8?q?=E6=A0=87=E5=87=86=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/data_handle/TownStandardBatch.groovy | 251 ++++++++++++++++++ 1 file changed, 251 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy new file mode 100644 index 000000000..afe90bbb2 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy @@ -0,0 +1,251 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.BaseBatch +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.xiamen.ElasticUtils89 +import com.mongodb.BasicDBObject +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.update.UpdateRequest +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.common.xcontent.XContentType + +/** + * 区域标准化 + * 1.小区库乡镇街道标准化,包含小区,商铺,写字楼 + * 2.poi数据库中乡镇街道标准化 + */ +class TownStandardBatch extends BaseBatch{ + + def fullCityMap = CommonDatautils.queryFullNameCity() + def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity() + def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity() + + static main(args) { + runBatch(new TownStandardBatch(), args) + } + + + @Override + void process(Object args) { + TownStandard() + } + + def TownStandard(){ + townStandardByCat("office") +// townStandardByPoi() + } + def townStandardByCat(def cat){ + //"三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德", + def citys = CommonDatautils.findAllCity() + for(def city : citys){ + + def data = fullCityMap.get(city) + def fullCityName = null + if(data){ + fullCityName = data.city + } + + def mzbRegionMap = CommonDatautils.queryMzbRegion(fullCityName) + def shortRegionNameMap = CommonDatautils.getRegionShortName(fullCityName) + def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district", "street", "alias"], mzbRegionMap) + if(townMap){ + def regionSet = townMap.keySet() + if(regionSet.size()>1){ + haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap) + }else{ + println city +" 无区域" + } + } + } + } + + def townStandardByPoi(){ + def citys = ["重庆"]//CommonDatautils.findAllCity() + for(def city : citys){ + + def data = fullCityMap.get(city) + def fullCityName = null + if(data){ + fullCityName = data.city + } + if(fullCityName){ + + def rMap = regionFullNameMap.get(fullCityName) + if(rMap){ + def set = rMap.keySet() + def rets = findEsPoiByCity(fullCityName) + println city +" "+rets.size() + if(set.size()>1){ + poiHaveRegionProcess(city, set, rMap, rets) + }else{ +// haveStreetProcess(city, set, cat) + } + } + }else{ + println "没找到相应的全称:"+city + } + } + } + def haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap){ + def results = [] + if(fullCityName){ + def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name", "township"]) + for(def d : districts){ + if(d.township){ + def towns = null + def updateTown = null + for(def m : townMap){ + def fullRegionName = shortRegionNameMap.get(d.region) + if(m.key == fullRegionName){ + towns = m.value + break + } + } + + def opt = true + if(towns){ + for(def town : towns){ + def townAlias = [] + if(town.alias){ + townAlias = town.alias + } + townAlias << town.street + if(townAlias.contains(d.township)){ + if(d.township!=town.street){ + updateTown = town.street + }else{ + opt = false + } + break + } + } + } + if(opt){ + if(updateTown){ + def udata = new BasicDBObject() + udata.put("township", updateTown) + udata.put("m_date", new Date()) + println d.city +" "+d.region+" "+d.name+" "+d.township +" 乡镇修改为:"+updateTown +// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + }else{ + def udata = new BasicDBObject() + udata.put("township", "") + udata.put("m_date", new Date()) + println d.city +" "+d.region+" "+d.name+" "+d.township +" 乡镇修改为:空值" +// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + } + } + + } + } + } + } + + def updateRegionByEs(rows){ + def client = ElasticUtils89.getClient("192.168.0.89") + def bulkRequest = new BulkRequest() + + rows.each { row -> + def id = row.keySet().iterator().next() + def doc = row.get(id) + + def updateRequest = new UpdateRequest("didata_map_poi", id) + updateRequest.doc(doc, XContentType.JSON) + bulkRequest.add(updateRequest) + } + + try { + // 执行批量更新请求 + def bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT) + + // 检查响应 + if (bulkResponse.hasFailures()) { + println("批量更新中有失败的操作:") + bulkResponse.buildFailureMessage().each { println(it) } + } else { + println("批量更新成功!") + } + } catch (Exception e) { + e.printStackTrace() + } finally { + // 关闭客户端 + try { + client.close() + } catch (Exception e) { + e.printStackTrace() + } + } + } + + + // def queryPoiDatasByCity(){ +// def esUrl = "http://localhost:9200/your_index/_search" +// def pageSize = 1000 // 每页返回的记录数 +// def scrollSize = '1m' // scroll上下文的生存时间 +// +//// 初始化HttpClient和ObjectMapper +// CloseableHttpClient httpClient = HttpClients.createDefault() +// ObjectMapper objectMapper = new ObjectMapper() +// +//// 初始化scroll请求 +// def scrollRequest = """ +//{ +// "size": $pageSize, +// "query": { +// "match_all": {} +// }, +// "scroll": "$scrollSize" +//} +//""" +// +//// 发送scroll初始化请求 +// HttpGet scrollInitRequest = new HttpGet(esUrl) +// scrollInitRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequest, "UTF-8")) +// scrollInitRequest.setHeader("Content-Type", "application/json") +// +// HttpResponse scrollInitResponse = httpClient.execute(scrollInitRequest) +// HttpEntity entity = scrollInitResponse.getEntity() +// String scrollId = objectMapper.readValue(EntityUtils.toString(entity), Map.class).get("_scroll_id").toString() +// +//// 存储所有结果 +// def allResults = [] +// +//// 发送scroll请求获取数据,直到没有更多数据为止 +// while (true) { +// def scrollRequestWithId = """ +// { +// "scroll_id": "$scrollId", +// "scroll": "$scrollSize" +// } +// """ +// +// HttpGet scrollGetRequest = new HttpGet(esUrl + "?scroll") +// scrollGetRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequestWithId, "UTF-8")) +// scrollGetRequest.setHeader("Content-Type", "application/json") +// +// HttpResponse scrollGetResponse = httpClient.execute(scrollGetRequest) +// entity = scrollGetResponse.getEntity() +// def responseMap = objectMapper.readValue(EntityUtils.toString(entity), Map.class) +// +// def hits = responseMap.get("hits").get("hits") +// if (hits.isEmpty()) { +// break // 没有更多数据,跳出循环 +// } +// +// // 合并当前页的数据 +// allResults.addAll(hits.map({ it.get("_source") })) +// +// // 更新scrollId以进行下一次请求 +// scrollId = responseMap.get("_scroll_id").toString() +// } +// +// // 关闭HttpClient +// httpClient.close() +// +// // 输出或处理全量数据 +// println(allResults) +// +// // 注意:实际应用中,你可能需要在获取完数据后清除scroll上下文,以释放资源。 +// } + +} From 9fcc8b38578444a83fbef8f8f4571e4bda3c937e Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 1 Nov 2024 11:10:46 +0800 Subject: [PATCH 25/35] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=9F=A5=E5=8C=BA?= =?UTF-8?q?=E5=9F=9F=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/commData/CommonDatautils.groovy | 119 +++++++++++++----- 1 file changed, 85 insertions(+), 34 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index cc8d27021..667580f1d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1,43 +1,22 @@ package com.fangjia.script.commData import com.fangjia.core.RegionData -import com.fangjia.datastore.driver.FacetResult -import com.fangjia.httpclient.UserAgents -import com.fangjia.script.common.DataHandleUtils -import com.fangjia.script.looplineinfo.InsidePolygon -import com.fangjia.script.util.DbUtils -import com.fangjia.script.util.ElasticUtils -import com.fangjia.script.util.JacksonUtil -import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils -import com.mongodb.AggregationOutput -import com.mongodb.client.MongoCursor -import com.mongodb.client.model.Accumulators -import com.mongodb.client.model.Aggregates -import com.rabbitmq.client.impl.AMQImpl -import groovy.json.JsonSlurper - -import net.sf.json.JSONArray -import net.sf.json.JSONObject -import org.apache.commons.lang.StringUtils -import org.apache.poi.ss.util.RegionUtil -import org.bson.Document -import org.bson.types.ObjectId - -import com.fangjia.core.util.HttpRequest; +import com.fangjia.core.util.HttpRequest import com.fangjia.datastore.driver.DSCollection import com.fangjia.datastore.driver.DSCursor import com.fangjia.datastore.driver.DataStore +import com.fangjia.httpclient.UserAgents +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.looplineinfo.InsidePolygon import com.fangjia.script.newDistrict.InvokeHttpRequest -import com.fangjia.script.util.DateUtils -import com.fangjia.script.util.GeoHelper -import com.fangjia.script.util.GeoUtils -import com.fangjia.script.util.MongoUtils -import com.mongodb.BasicDBObject -import com.mongodb.Bytes -import com.mongodb.DBCollection -import com.mongodb.DBCursor -import com.mongodb.DBObject -import com.mongodb.QueryOperators +import com.fangjia.script.util.* +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.* +import groovy.json.JsonSlurper +import net.sf.json.JSONArray +import net.sf.json.JSONObject +import org.apache.commons.lang.StringUtils +import org.bson.types.ObjectId import org.elasticsearch.index.query.BoolQueryBuilder import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders @@ -3546,6 +3525,20 @@ class CommonDatautils { return prices; } + static def queryCityPriceByWeekend(city, weekend){ + DBObject queryCondition = new BasicDBObject() + queryCondition.put("city", city); + queryCondition.put("cat", "城市"); + + queryCondition.put('weekend', DateUtils.parseFromTightYYYYMMDD(weekend)); + def obj = REGION_STAT_COLL.findOne(queryCondition) + def prices = null; + if(obj) { + prices = obj.get("avg_price"); + } + return prices; + } + @@ -4838,6 +4831,24 @@ class CommonDatautils { return retMap } + static def getRegionShortName(city){ + def q = new BasicDBObject() + q.put("type","区域") + q.put("district_diff",1) + q.put("city", city) + + def keys = new BasicDBObject() + keys.put("district",1) + keys.put("district_short", 1) + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def shortNameMap = [:] + for(def d : dbcursor){ + shortNameMap.put(d.district_short, d.district) + } + return shortNameMap + } + + static def queryAllRegionFullNameByCity(){ def q = new BasicDBObject() q.put("type","区域") @@ -4909,7 +4920,47 @@ class CommonDatautils { return townMap } + static def queryMzbRegion(city){ + def q = new BasicDBObject() + q.put("type","区域") + q.put("city",city) + def keys = new BasicDBObject() + keys.put("district",1) + keys.put("mzb_district_name",1) + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def regionMap = [:] + for(def d : dbcursor){ + regionMap.put(d.district, d.mzb_district_name) + } + return regionMap + } + + static def queryTownByCity(city, fields, mzbRegionMap){ + + def q = new BasicDBObject() + q.put("type","乡镇街道") + q.put("city",city) + + def keys = new BasicDBObject() + for(def key : fields){ + keys.put(key,1) + } + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def townMap = [:] + for(def d : dbcursor){ + def key = mzbRegionMap.get(d.district) + def list = townMap.get(key) + if(!list){ + list = [] + } + list << d + townMap.put(key, list) + } + + println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss") + return townMap + } static def queryTownByCity(city){ @@ -4945,7 +4996,7 @@ class CommonDatautils { def q = new BasicDBObject() q.put("type","乡镇街道") - q.put("city",new BasicDBObject('$in', ["深圳"])) + def keys = new BasicDBObject() keys.put("city",1) From fac47d1faf38fa6af157ee8bca4a810cb10b8290 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 1 Nov 2024 14:37:38 +0800 Subject: [PATCH 26/35] =?UTF-8?q?=E6=8C=87=E6=95=B0=E8=84=9A=E6=9C=AC?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/trade/ImportIndexStats.groovy | 146 ++++++++++-------- 1 file changed, 78 insertions(+), 68 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy index b5de80cae..7e5d2151c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy @@ -1,16 +1,14 @@ package com.fangjia.script.trade import com.fangjia.core.util.DataUtil -import com.fangjia.core.util.DateUtil import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher ///导入价格指数 -import com.fangjia.script.BatchOwner -import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils -import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.MongoUtils @@ -19,10 +17,8 @@ import com.mongodb.DBCollection import com.mongodb.DBObject import org.apache.commons.lang.StringUtils import org.apache.commons.lang.math.RandomUtils -import org.bson.types.ObjectId import org.elasticsearch.xpack.sql.jdbc.EsDataSource -import java.sql.ResultSet import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors @@ -74,13 +70,19 @@ class ImportIndexStats extends BaseBatch{ // updateDataType() // synch() -// synchTjjIndex()//同步统计局抓取过来的指数 +// synchTjjIndex()//批量同步统计局抓取过来的指数 Calendar calendar=Calendar.getInstance(); def month= calendar.get(Calendar.MONTH); def year= calendar.get(Calendar.YEAR); def date = createDateString(year, month, 0, ""); - calculatePriceIndex(year, month, date); + + def day = calendar.get(Calendar.DATE) + if(day == 5){ + calculatePriceIndex(year, month, date); + } + + createTjjData(date)//同步单期统计局数据 // addLostIndex() // addHistoryIndex() @@ -199,7 +201,7 @@ class ImportIndexStats extends BaseBatch{ runDates << map s = DateUtils.addMonth(date, 1, "yyyyMM") - if(year==2024 && month==8){ + if(year==2024 && month==10){ stop = true; } } @@ -210,7 +212,7 @@ class ImportIndexStats extends BaseBatch{ //计算统计局没有发布的城市的价格指数 def calculatePriceIndex(def year, def month, def date){ -// def citys = ["五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]// +// def citys = ["定安","五指山","琼海","儋州","文昌","万宁","东方","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]// def citys = CommonDatautils.findAllCity(); // def date = createDateString(year, month, 0, ""); def haveIndexCitys = findPriceIndexCitys(date.toInteger()); @@ -233,14 +235,14 @@ class ImportIndexStats extends BaseBatch{ if(id){//已计算 continue; } - if(date.toInteger()>=201601){ - dingjiYear = 2015 - }else if(date.toInteger()>=202101){ - dingjiYear = 2020 + if(date.toInteger()>=203101){ + dingjiYear = 2030 }else if(date.toInteger()>=202601){ dingjiYear = 2025 - }else if(date.toInteger()>=203101){ - dingjiYear = 2030 + }else if(date.toInteger()>=202101){ + dingjiYear = 2020 + }else if(date.toInteger()>=201601){ + dingjiYear = 2015 } fixedThreadPool.execute(new Runnable() { void run() { @@ -259,6 +261,7 @@ class ImportIndexStats extends BaseBatch{ }); } latch.await(1, TimeUnit.MINUTES); +// latch.await(1, TimeUnit.MINUTES); } def calculating(city, year, month, date){ @@ -826,69 +829,76 @@ class ImportIndexStats extends BaseBatch{ } def createTjjData(date){ - def queryStartDate = DateUtils.parseFromTightByPattern(date+"01", "yyyyMMdd") - def queryEndDate = DateUtils.parseFromTightByPattern(date+"05", "yyyyMMdd") - def sellIndexMap = findTjjIndex(indices_price_house_sellColl, queryStartDate, queryEndDate) - def sellIndexAreaMap = findTjjAreaIndex(indices_price_house_sell_typeColl, queryStartDate, queryEndDate) - def newIndexMap = findTjjIndex(indices_price_house_newColl, queryStartDate, queryEndDate) - def newAreaIndexAreaMap = findTjjAreaIndex(indices_price_house_new_typeColl, queryStartDate, queryEndDate) - def rets = [] - sellIndexMap.each{city, d-> - def dbo = new BasicDBObject() - dbo.put("city",d.city) - dbo.put("yearmonth",date.toInteger()) - dbo.put("source" ,1) - dbo.put("remark" , "统计局数据导入") - dbo.put("cat" , "district") - dbo.put("c_date" , new Date()) - dbo.put("m_date" , new Date()) - dbo.put("type" , "城市") - dbo.put("chain" ,d.mom) - dbo.put("yeartoyear" , d.yoy) - dbo.put("fixedbase" , d.fixedbase) + def count = STATS_INDEX_COLL.count(new BasicDBObject("type","城市").append("source", 1).append("yearmonth", date.toInteger())) + if(count == 0){ + def queryStartDate = DateUtils.parseFromTightByPattern(date+"01", "yyyyMMdd") + def queryEndDate = DateUtils.parseFromTightByPattern(date+"05", "yyyyMMdd") + def sellIndexMap = findTjjIndex(indices_price_house_sellColl, queryStartDate, queryEndDate) + def sellIndexAreaMap = findTjjAreaIndex(indices_price_house_sell_typeColl, queryStartDate, queryEndDate) + + def newIndexMap = findTjjIndex(indices_price_house_newColl, queryStartDate, queryEndDate) + def newAreaIndexAreaMap = findTjjAreaIndex(indices_price_house_new_typeColl, queryStartDate, queryEndDate) + def rets = [] + sellIndexMap.each{city, d-> + def dbo = new BasicDBObject() + dbo.put("city",d.city) + dbo.put("yearmonth",date.toInteger()) + dbo.put("source" ,1) + dbo.put("remark" , "统计局数据导入") + dbo.put("cat" , "district") + dbo.put("c_date" , new Date()) + dbo.put("m_date" , new Date()) + dbo.put("type" , "城市") + dbo.put("chain" ,d.mom) + dbo.put("yeartoyear" , d.yoy) + dbo.put("fixedbase" , d.fixedbase) - def areaMap = sellIndexAreaMap.get(d.city) - def areaObj1 = areaMap.get("90") - dbo.put("below_chain" , areaObj1.mom) - dbo.put("below_yeartoyear" , areaObj1.yoy) - dbo.put("below_fixedbase" , areaObj1.fixedbase) + def areaMap = sellIndexAreaMap.get(d.city) + def areaObj1 = areaMap.get("90") + dbo.put("below_chain" , areaObj1.mom) + dbo.put("below_yeartoyear" , areaObj1.yoy) + dbo.put("below_fixedbase" , areaObj1.fixedbase) - def areaObj2 = areaMap.get("90-144") - dbo.put("among_chain" , areaObj2.mom) - dbo.put("among_yeartoyear" , areaObj2.yoy) - dbo.put("among_fixedbase" , areaObj2.fixedbase) + def areaObj2 = areaMap.get("90-144") + dbo.put("among_chain" , areaObj2.mom) + dbo.put("among_yeartoyear" , areaObj2.yoy) + dbo.put("among_fixedbase" , areaObj2.fixedbase) - def areaObj3 = areaMap.get("144") - dbo.put("above_chain" , areaObj3.mom) - dbo.put("above_yeartoyear" , areaObj3.yoy) - dbo.put("above_fixedbase" , areaObj3.fixedbase) + def areaObj3 = areaMap.get("144") + dbo.put("above_chain" , areaObj3.mom) + dbo.put("above_yeartoyear" , areaObj3.yoy) + dbo.put("above_fixedbase" , areaObj3.fixedbase) - def newObj = newIndexMap.get(d.city) - dbo.put("new_chain" , newObj.mom) - dbo.put("new_yeartoyear" , newObj.yoy) - dbo.put("new_fixedbase" , newObj.fixedbase) + def newObj = newIndexMap.get(d.city) + dbo.put("new_chain" , newObj.mom) + dbo.put("new_yeartoyear" , newObj.yoy) + dbo.put("new_fixedbase" , newObj.fixedbase) - def newAreaMap = newAreaIndexAreaMap.get(d.city) - def areaNewObj1 = newAreaMap.get("90") - dbo.put("new_below_chain" , areaNewObj1.mom) - dbo.put("new_below_yeartoyear" , areaNewObj1.yoy) - dbo.put("new_below_fixedbase" , areaNewObj1.fixedbase) + def newAreaMap = newAreaIndexAreaMap.get(d.city) + def areaNewObj1 = newAreaMap.get("90") + dbo.put("new_below_chain" , areaNewObj1.mom) + dbo.put("new_below_yeartoyear" , areaNewObj1.yoy) + dbo.put("new_below_fixedbase" , areaNewObj1.fixedbase) - def areaNewObj2 = newAreaMap.get("90-144") - dbo.put("new_among_chain" , areaNewObj2.mom) - dbo.put("new_among_yeartoyear" , areaNewObj2.yoy) - dbo.put("new_among_fixedbase" , areaNewObj2.fixedbase) + def areaNewObj2 = newAreaMap.get("90-144") + dbo.put("new_among_chain" , areaNewObj2.mom) + dbo.put("new_among_yeartoyear" , areaNewObj2.yoy) + dbo.put("new_among_fixedbase" , areaNewObj2.fixedbase) - def areaNewObj3 = newAreaMap.get("144") - dbo.put("new_above_chain" , areaNewObj3.mom) - dbo.put("new_above_yeartoyear" , areaNewObj3.yoy) - dbo.put("new_above_fixedbase" , areaNewObj3.fixedbase) + def areaNewObj3 = newAreaMap.get("144") + dbo.put("new_above_chain" , areaNewObj3.mom) + dbo.put("new_above_yeartoyear" , areaNewObj3.yoy) + dbo.put("new_above_fixedbase" , areaNewObj3.fixedbase) + + rets << dbo + } + if(rets){ + STATS_INDEX_COLL.insert(rets) + } - rets << dbo } - STATS_INDEX_COLL.insert(rets) } def findTjjIndex(coll, queryStartDate, queryEndDate){ From 482b620029c45fd4ea7d21f3dbc16d68fd65f00c Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 1 Nov 2024 17:52:04 +0800 Subject: [PATCH 27/35] =?UTF-8?q?=E5=85=A8=E9=83=A8=E5=9F=8E=E5=B8=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/data_handle/RegionStandardBatch.groovy | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy index c4ffa723f..12213a4b9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy @@ -29,18 +29,20 @@ class RegionStandardBatch extends BaseBatch{ @Override void process(Object args) { -// seaweedRegionStandard() - poiRegionStandard() + seaweedRegionStandard() +// poiRegionStandard() } def seaweedRegionStandard(){ - seaweedRegionStandardByCat("shop") + seaweedRegionStandardByCat("district") } def poiRegionStandard(){ - //"三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德", - def citys = ["重庆"]//CommonDatautils.findAllCity() + def runCitys = ["重庆","三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德"] + def citys = CommonDatautils.findAllCity() for(def city : citys){ - + if(runCitys.contains(city)){ + continue + } def data = fullCityMap.get(city) def fullCityName = null if(data){ From 10f0b7908722c5d62e15fc7f0a1fdf8a07399b57 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 4 Nov 2024 16:45:39 +0800 Subject: [PATCH 28/35] =?UTF-8?q?=E6=9F=A5poi=E6=95=B0=E6=8D=AE=E9=99=90?= =?UTF-8?q?=E5=88=B6=E8=BF=94=E5=9B=9E=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/data_handle/RegionStandardBatch.groovy | 7 +- .../gj/data_handle/TownStandardBatch.groovy | 108 ++++++------------ .../script/gj/xiamen/ElasticUtils89.groovy | 10 +- 3 files changed, 48 insertions(+), 77 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy index 12213a4b9..9e30b064c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy @@ -29,8 +29,8 @@ class RegionStandardBatch extends BaseBatch{ @Override void process(Object args) { - seaweedRegionStandard() -// poiRegionStandard() +// seaweedRegionStandard() + poiRegionStandard() } def seaweedRegionStandard(){ @@ -168,7 +168,8 @@ class RegionStandardBatch extends BaseBatch{ QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("district", "蕉岭县")) def rets = [] - def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi", queryBuilder) + String[] includes = ["city", "district", "name", "township", "status", "id"] + def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi", queryBuilder,includes) for(def d: datas){ def o = new BasicDBObject() o.put("city", d.city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy index afe90bbb2..d2c6381ae 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.gj.data_handle + import com.fangjia.script.BaseBatch import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.xiamen.ElasticUtils89 @@ -8,6 +9,8 @@ import org.elasticsearch.action.bulk.BulkRequest import org.elasticsearch.action.update.UpdateRequest import org.elasticsearch.client.RequestOptions import org.elasticsearch.common.xcontent.XContentType +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders /** * 区域标准化 @@ -20,6 +23,13 @@ class TownStandardBatch extends BaseBatch{ def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity() def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity() + def poiEsUrl = "http://192.168.0.89:9200/didata_map_poi/_search" + + def elasticsearchUrl = "http://192.168.0.89:9200/_search/scroll" + def initialSearchUrl = "http://192.168.0.89:9200/didata_map_poi/_search?scroll=1m" + def scrollTime = "1m" // Scroll context time-to-live + def pageSize = 1000 // Number of hits to return per scroll + static main(args) { runBatch(new TownStandardBatch(), args) } @@ -28,11 +38,13 @@ class TownStandardBatch extends BaseBatch{ @Override void process(Object args) { TownStandard() + + } def TownStandard(){ - townStandardByCat("office") -// townStandardByPoi() +// townStandardByCat("office") + townStandardByPoi() } def townStandardByCat(def cat){ //"三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德", @@ -60,7 +72,7 @@ class TownStandardBatch extends BaseBatch{ } def townStandardByPoi(){ - def citys = ["重庆"]//CommonDatautils.findAllCity() + def citys = ["三明"]//CommonDatautils.findAllCity() for(def city : citys){ def data = fullCityMap.get(city) @@ -141,6 +153,9 @@ class TownStandardBatch extends BaseBatch{ } } + + + def updateRegionByEs(rows){ def client = ElasticUtils89.getClient("192.168.0.89") def bulkRequest = new BulkRequest() @@ -177,75 +192,24 @@ class TownStandardBatch extends BaseBatch{ } } + def findEsPoiByCity(city){ + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) + def rets = [] + + String[] includes = ["city", "district", "name", "township", "status", "id"] + def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi", queryBuilder, includes) + for(def d: datas){ + def o = new BasicDBObject() + o.put("city", d.city) + o.put("district", d.district) + o.put("name", d.name) + o.put("township", d.township) + o.put("status", d.status) + o.put("id", d.id) + rets << o + } + return rets + } - // def queryPoiDatasByCity(){ -// def esUrl = "http://localhost:9200/your_index/_search" -// def pageSize = 1000 // 每页返回的记录数 -// def scrollSize = '1m' // scroll上下文的生存时间 -// -//// 初始化HttpClient和ObjectMapper -// CloseableHttpClient httpClient = HttpClients.createDefault() -// ObjectMapper objectMapper = new ObjectMapper() -// -//// 初始化scroll请求 -// def scrollRequest = """ -//{ -// "size": $pageSize, -// "query": { -// "match_all": {} -// }, -// "scroll": "$scrollSize" -//} -//""" -// -//// 发送scroll初始化请求 -// HttpGet scrollInitRequest = new HttpGet(esUrl) -// scrollInitRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequest, "UTF-8")) -// scrollInitRequest.setHeader("Content-Type", "application/json") -// -// HttpResponse scrollInitResponse = httpClient.execute(scrollInitRequest) -// HttpEntity entity = scrollInitResponse.getEntity() -// String scrollId = objectMapper.readValue(EntityUtils.toString(entity), Map.class).get("_scroll_id").toString() -// -//// 存储所有结果 -// def allResults = [] -// -//// 发送scroll请求获取数据,直到没有更多数据为止 -// while (true) { -// def scrollRequestWithId = """ -// { -// "scroll_id": "$scrollId", -// "scroll": "$scrollSize" -// } -// """ -// -// HttpGet scrollGetRequest = new HttpGet(esUrl + "?scroll") -// scrollGetRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequestWithId, "UTF-8")) -// scrollGetRequest.setHeader("Content-Type", "application/json") -// -// HttpResponse scrollGetResponse = httpClient.execute(scrollGetRequest) -// entity = scrollGetResponse.getEntity() -// def responseMap = objectMapper.readValue(EntityUtils.toString(entity), Map.class) -// -// def hits = responseMap.get("hits").get("hits") -// if (hits.isEmpty()) { -// break // 没有更多数据,跳出循环 -// } -// -// // 合并当前页的数据 -// allResults.addAll(hits.map({ it.get("_source") })) -// -// // 更新scrollId以进行下一次请求 -// scrollId = responseMap.get("_scroll_id").toString() -// } -// -// // 关闭HttpClient -// httpClient.close() -// -// // 输出或处理全量数据 -// println(allResults) -// -// // 注意:实际应用中,你可能需要在获取完数据后清除scroll上下文,以释放资源。 -// } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy index e31ded3ec..a401bf480 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy @@ -26,6 +26,7 @@ import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.search.Scroll import org.elasticsearch.search.SearchHit import org.elasticsearch.search.builder.SearchSourceBuilder +import org.elasticsearch.search.fetch.subphase.FetchSourceContext import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.Connection @@ -35,7 +36,6 @@ import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.TimeUnit import java.util.function.BiConsumer - /** * 功能说明:Elasticsearch工具类 */ @@ -228,13 +228,19 @@ class ElasticUtils89 { } - static def executeSimpleQuery(String host, String indexName, QueryBuilder qb){ + static def executeSimpleQuery(String host, String indexName, QueryBuilder qb, includes){ final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L)) RestHighLevelClient client = getClient(host) SearchRequest searchRequest = new SearchRequest(indexName); searchRequest.scroll(scroll); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + if(includes){ + String[] excludes = [] // 没有要排除的字段 + FetchSourceContext fetchSourceContext = new FetchSourceContext(true, includes, excludes) + searchSourceBuilder.fetchSource(fetchSourceContext) + } + searchSourceBuilder.query(qb); // searchSourceBuilder.size(1000) searchRequest.source(searchSourceBuilder); From 05b8c6814b9138dc04da5587eba6cdd0b7a2e740 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 4 Nov 2024 16:57:01 +0800 Subject: [PATCH 29/35] =?UTF-8?q?=E5=8E=BB=E6=8E=89=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/gj/data_handle/RegionStandardBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy index 9e30b064c..30d1e0143 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy @@ -165,7 +165,7 @@ class RegionStandardBatch extends BaseBatch{ } def findEsPoiByCity(city){ - QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("district", "蕉岭县")) + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) def rets = [] String[] includes = ["city", "district", "name", "township", "status", "id"] From 4c95a74d60ca5bab1b2e814580e841b7f4446030 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 4 Nov 2024 20:02:12 +0800 Subject: [PATCH 30/35] =?UTF-8?q?poi=E8=A1=97=E9=81=93=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/commData/CommonDatautils.groovy | 25 ++++++ .../gj/data_handle/TownStandardBatch.groovy | 76 +++++++++++++++++-- 2 files changed, 95 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 667580f1d..15c76f043 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -4962,6 +4962,31 @@ class CommonDatautils { return townMap } + static def queryTownByCity(city, fields){ + + def q = new BasicDBObject() + q.put("type","乡镇街道") + q.put("city",city) + + def keys = new BasicDBObject() + for(def key : fields){ + keys.put(key,1) + } + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def townMap = [:] + for(def d : dbcursor){ + def list = townMap.get(d.district) + if(!list){ + list = [] + } + list << d + townMap.put(d.district, list) + } + + println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss") + return townMap + } + static def queryTownByCity(city){ def q = new BasicDBObject() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy index d2c6381ae..1cbbde65c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy @@ -38,8 +38,6 @@ class TownStandardBatch extends BaseBatch{ @Override void process(Object args) { TownStandard() - - } def TownStandard(){ @@ -81,14 +79,14 @@ class TownStandardBatch extends BaseBatch{ fullCityName = data.city } if(fullCityName){ - + def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district", "street", "alias"]) def rMap = regionFullNameMap.get(fullCityName) if(rMap){ def set = rMap.keySet() def rets = findEsPoiByCity(fullCityName) println city +" "+rets.size() if(set.size()>1){ - poiHaveRegionProcess(city, set, rMap, rets) + poiHaveRegionProcess(fullCityName, townMap, rets) }else{ // haveStreetProcess(city, set, cat) } @@ -153,10 +151,76 @@ class TownStandardBatch extends BaseBatch{ } } + def poiHaveRegionProcess(fullCityName, townMap, rets){ + def results = [] + if(fullCityName){ + for(def d : rets){ + if(d.township){ + def towns = null + def updateTown = null + for(def m : townMap){ + def townDatas = townMap.get(d.district) + if(townDatas){ + towns = townDatas + break + } + } + + def opt = true + if(towns){ + for(def town : towns){ + def townAlias = [] + if(town.alias){ + townAlias = town.alias + } + townAlias << town.street + if(townAlias.contains(d.township)){ + if(d.township!=town.street){ + updateTown = town.street + }else{ + opt = false + } + break + } + } + } + if(opt){ + if(updateTown){ + def m = [:] + m.put(d.id, ["township":updateTown]) + results << m +// if(results.size()%100==0){ +// updateByEs(results) +// results.clear() +// } + println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown + }else{ + def m = [:] + m.put(d.id, ["township":""]) + results << m +// if(results.size()%100==0){ +// updateByEs(results) +// results.clear() +// } + println d.city +" "+d.district+" "+d.name +" "+d.township+" 状态修改为:空值" + } + } + + } + } + } + + if(results.size()>0){ + updateByEs(results) + results.clear() + } + println fullCityName +" 处理完成" + } - def updateRegionByEs(rows){ + + def updateByEs(rows){ def client = ElasticUtils89.getClient("192.168.0.89") def bulkRequest = new BulkRequest() @@ -193,7 +257,7 @@ class TownStandardBatch extends BaseBatch{ } def findEsPoiByCity(city){ - QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("status", 0)) def rets = [] String[] includes = ["city", "district", "name", "township", "status", "id"] From 6d752d945e28ed5904297da5408dd38b82c47d66 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 5 Nov 2024 09:14:11 +0800 Subject: [PATCH 31/35] update --- .../script/gj/data_handle/RegionStandardBatch.groovy | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy index 30d1e0143..7536d908f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy @@ -38,8 +38,13 @@ class RegionStandardBatch extends BaseBatch{ } def poiRegionStandard(){ def runCitys = ["重庆","三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德"] + def index = 0 def citys = CommonDatautils.findAllCity() for(def city : citys){ + index ++ + if(index<32){ + continue + } if(runCitys.contains(city)){ continue } @@ -53,13 +58,15 @@ class RegionStandardBatch extends BaseBatch{ def rMap = regionFullNameMap.get(fullCityName) if(rMap){ def set = rMap.keySet() - def rets = findEsPoiByCity(fullCityName) - println city +" "+rets.size() if(set.size()>1){ + def rets = findEsPoiByCity(fullCityName) + println city +" "+rets.size() poiHaveRegionProcess(city, set, rMap, rets) }else{ // haveStreetProcess(city, set, cat) } + }else{ + println "没找区域:"+city } }else{ println "没找到相应的全称:"+city From 3cb884d042049318d67e92be994af5b35e3ec4d2 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 5 Nov 2024 09:15:36 +0800 Subject: [PATCH 32/35] update --- .../script/gj/data_handle/DevUpdateTestBatch.groovy | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy index cf1b49203..49d2547b7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy @@ -34,8 +34,8 @@ class DevUpdateTestBatch { static main(args) { def task = new DevUpdateTestBatch() def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx - task.addDev(path) -// task.addDevAddress(path) +// task.addDev(path) + task.addDevAddress(path) // // task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是") @@ -122,6 +122,8 @@ class DevUpdateTestBatch { q.put("address", d['C']) }else if(d['B'] == "园区"){ q.put("park", d['C']) + }else if(d['B'] == "Poi"){ + q.put("poi", d['C']) } def mappingObj = DEV_MAPPING_100_CONN.findOne(q) if(!mappingObj){ @@ -137,6 +139,8 @@ class DevUpdateTestBatch { o.put("address", d['C']) }else if(d['B'] == "园区"){ o.put("park", d['C']) + }else if(d['B'] == "Poi"){ + o.put("poi", d['C']) } def date = new Date() o.put("c_date", date) @@ -164,7 +168,7 @@ class DevUpdateTestBatch { println d['A'] +" 不存在" } } - },1) + },2) } From 4bfbf97ac01b4f112ef5f9f52b1a3a32d9e0978b Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 5 Nov 2024 10:00:58 +0800 Subject: [PATCH 33/35] =?UTF-8?q?=E7=AC=AC=E4=B8=89=E6=96=B9=E9=87=8D?= =?UTF-8?q?=E6=96=B0=E5=8C=B9=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy index 5da686848..329a548fb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy @@ -44,7 +44,7 @@ class ThirdPriceMappingBatch extends BaseBatch{ @Override public void process(Object args) { // //将新增数据同步过来 - addNemCommunity() +// addNemCommunity() // //接口进行匹配 match() From 45b9678044428231e2945dd8077d7d1a5b5f03c3 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 5 Nov 2024 13:44:35 +0800 Subject: [PATCH 34/35] =?UTF-8?q?=E4=B9=A1=E9=95=87=E8=A1=97=E9=81=93?= =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/gj/data_handle/TownStandardBatch.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy index 1cbbde65c..92fdf5efe 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy @@ -46,7 +46,7 @@ class TownStandardBatch extends BaseBatch{ } def townStandardByCat(def cat){ //"三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德", - def citys = CommonDatautils.findAllCity() + def citys = ["重庆","龙岩","南平","漳州","福州","厦门","泉州","莆田","三明","宁德"]//CommonDatautils.findAllCity() for(def city : citys){ def data = fullCityMap.get(city) @@ -70,7 +70,7 @@ class TownStandardBatch extends BaseBatch{ } def townStandardByPoi(){ - def citys = ["三明"]//CommonDatautils.findAllCity() + def citys = ["宁德"]//CommonDatautils.findAllCity() for(def city : citys){ def data = fullCityMap.get(city) From 846061a61f91e49936d683dcc87289b766cf9ff1 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 5 Nov 2024 14:12:01 +0800 Subject: [PATCH 35/35] =?UTF-8?q?=E5=9F=8E=E5=B8=82=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/gj/data_handle/TownStandardBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy index 92fdf5efe..95ecf6a9f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy @@ -70,7 +70,7 @@ class TownStandardBatch extends BaseBatch{ } def townStandardByPoi(){ - def citys = ["宁德"]//CommonDatautils.findAllCity() + def citys = citys = ["重庆","龙岩","南平","漳州","福州","厦门","泉州","莆田","三明","宁德"]//CommonDatautils.findAllCity() for(def city : citys){ def data = fullCityMap.get(city)