From 9f7a4d5ef3ace0cde2720fff2101c48e9640504a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 20 Jun 2024 18:40:13 +0800 Subject: [PATCH 1/9] =?UTF-8?q?=E6=88=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/BigdataDealBatch.groovy | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index bcd484c2b..2955ed1e6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -62,20 +62,20 @@ class BigdataDealBatch extends BaseBatch{ @Override public void process(Object args) { - def date = DateUtils.parseFromTightByPattern("2024-5-21", "yyyy-MM-dd") - def objectId = dateToObjectId(date) - println objectId +// def date = DateUtils.parseFromTightByPattern("2024-5-21", "yyyy-MM-dd") +// def objectId = dateToObjectId(date) +// println objectId //664a22000000000000000000 5-20 //664b73800000000000000000 5-21 def citys = CcbCommonUtils.getRunningCitys(args) -// -// println "curDate:"+curDate -// def allCitys = CommonDatautils.findAllCity() -// for(def city : allCitys){ -// addDealPriceMatch(city); -// } -// updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月 + + println "curDate:"+curDate + def allCitys = CommonDatautils.findAllCity() + for(def city : allCitys){ + addDealPriceMatch(city); + } + updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月 def rows = [] @@ -727,6 +727,9 @@ class BigdataDealBatch extends BaseBatch{ } } rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name, null); + if(!rMap){ + + } if(!rMap && t.address){ def flag = t.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷).+[弄号]$') if(flag){ From 3b3615b2bf4f5e757d2f0e2fcfea4f6af0bc19ff Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 20 Jun 2024 21:43:11 +0800 Subject: [PATCH 2/9] update --- .../script/ccbevaluation/AddSzDealData.groovy | 18 +++++++------- .../ccbevaluation/BigdataDealBatch.groovy | 24 ++++++++++++------- .../ccbevaluation/CcbCommonUtils.groovy | 3 ++- .../ccbevaluation/GuiDingPriceBatch.groovy | 2 +- .../script/ccbevaluation/TradeStatLog.groovy | 8 +++---- .../script/commData/CommonDatautils.groovy | 2 +- .../com/fangjia/script/gj/OpenApiUtils.groovy | 5 +++- .../district/SeaweedDataMatchProcess.groovy | 12 ++++------ 8 files changed, 42 insertions(+), 32 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy index cb4abf8cf..ff77ef91f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy @@ -9,7 +9,7 @@ import com.mongodb.DBCollection /** * 深圳公众号成交数据录入 */ -def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳2024年3月份二手房成交数据汇总.xls", 0, 1) +def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\20240619--深圳成交数据.xlsx", 0, 1) def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) @@ -72,19 +72,21 @@ for(def d : datas){ } - map.area = d['F'].toString().toDouble() - map.total_price = (int)(d['G'].toString().toDouble().toInteger()*10000) - map.avg_price = (int)(d['H'].toString().toDouble()) + map.area = d['F'].replace("平","").toString().toDouble() + map.total_price = (int)(d['G'].replace("万","").toString().toDouble().toInteger()*10000) + map.avg_price = (int)(d['H'].replace("元/平","").toString().toDouble()) if(d['I']){ - map.sell_price = (int)(d['I'].toString().toDouble()*10000) + map.sell_price = (int)(d['I'].replace("万","").toString().toDouble()*10000) } + map.trade_date = d['J'] // def trade_date = d['J'] // if(trade_date){ // map.trade_date = DateUtils.parseFromTightByPattern(trade_date,'yyyy/MM/dd') -// }else{ - def trade_date1 = DateUtils.getRandomDate("2024-03-01", "2024-03-31") - map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd") +// } +// else{ +// def trade_date1 = DateUtils.getRandomDate("2024-03-01", "2024-03-31") +// map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd") // } map.source = "深房家族" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 2955ed1e6..8bbea25d3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -35,7 +35,7 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataDealBatch extends BaseBatch{ - def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -24, "yyyyMM")); + def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); @@ -726,18 +726,24 @@ class BigdataDealBatch extends BaseBatch{ t.name = t.name +"商住楼" } } - rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name, null); - if(!rMap){ - - } - if(!rMap && t.address){ - def flag = t.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷).+[弄号]$') + def flag = false + def address = null + if(t.address){ + flag = t.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷).+[弄号]$') if(flag){ - rMap = SeaweedDataMatchProcess.doMatch(t.city, region, null, t.address); + address = t.address } } + + rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name, address); + if(!rMap && !t.name.toString().endsWith("小区")){ + rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name+"小区", address); + } + if(rMap && rMap.region==region){ CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id) + }else if(rMap && rMap.address && address && rMap.region!=region && address.contains(rMap.address)){ + CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id) }else { if (rMap) { println "区域不同" @@ -755,7 +761,7 @@ class BigdataDealBatch extends BaseBatch{ } } - ); + ) } latch.await() println city +" "+trades.size()+" "+new Date().toString() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 7a4062270..7dac52e9c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -634,7 +634,8 @@ class CcbCommonUtils { q.put("city", city) q.put("trade_date", new BasicDBObject('$gte', startDate)) q.put("match_id", new BasicDBObject('$exists', false)) -// q.put("district_name","中海寰宇天下") +// q.put("district_name","新明胡同") +// q.put("address","桂苑大道135号") def dbcursor = DEAL_PRICE_COLL.find(q) def total = dbcursor.count() def rets = new HashSet() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index c43a791b6..5c735e4c3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2024-05-24", "2024-06-08") + def prices = CcbCommonUtils.findPersonPrice(city, "2024-05-24", "2024-06-21") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index 7ab2fc7e4..57bdfdbd9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -40,8 +40,8 @@ class TradeStatLog { static main(args) { def task = new TradeStatLog() - task.run()//统计最近3个月抓取的成交数据量 -// task.exportTrade()//导出未入库成交 +// task.run()//统计最近3个月抓取的成交数据量 + task.exportTrade()//导出未入库成交 // task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) // task.averageStat() @@ -499,8 +499,8 @@ class TradeStatLog { def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"] def rets = [] rets << headers - def startDate = "2023-08-01" - def endDate = "2023-10-01" + def startDate = "2024-04-01" + def endDate = "2024-06-01" for(def city : citys){ def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index e9fa92613..3bdde3598 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -4679,7 +4679,7 @@ class CommonDatautils { def q = new BasicDBObject() q.put("type","乡镇街道") - q.put("province",new BasicDBObject('$in', ['福建省','重庆市'])) + q.put("province",new BasicDBObject('$in', ['浙江省','江苏省'])) def keys = new BasicDBObject() keys.put("province",1) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy index cbd2845ec..6d931d1cc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy @@ -13,12 +13,15 @@ import net.sf.json.groovy.JsonSlurper import org.apache.commons.lang.StringUtils class OpenApiUtils { - static def matchDistrict(def city, def address, def suggestDistrict, def preciseLevelflag){ + static def matchDistrict(def city, def address, def suggestDistrict, def preciseLevelflag, def suggestName){ address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","").replace("<","").replace(">","").replace("%","").replace("\"","") def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&suggestDistrict="+suggestDistrict+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1" if(preciseLevelflag){ url = url +"&preciseLevel=strict" } + if(suggestName){ + url = url +"&suggestName="+suggestName + } String content = new HttpRequest().get(url, "utf-8"); def rMap = [:] if(content){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy index c26ded24c..ba72b03de 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy @@ -36,18 +36,16 @@ class SeaweedDataMatchProcess { } } - if(!rMap && name){ - rMap = OpenApiUtils.matchDistrict(city, name, region, false) - } - - - if(!rMap && address){ + if(!rMap && address && name){ def flag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$') if(flag){ - rMap = OpenApiUtils.matchDistrict(city, address, region, true) + rMap = OpenApiUtils.matchDistrict(city, address, region, true, name) } } + if(!rMap && name){ + rMap = OpenApiUtils.matchDistrict(city, name, region, false, null) + } // if(rMap && rMap.id && region.contains(rMap.region)) { // def obj = CommonDatautils.findDistrictById(rMap.id.toString()); // if (obj) { From 28057687b3f688a6ee27aaf599d0a88a507bed07 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 21 Jun 2024 10:41:12 +0800 Subject: [PATCH 3/9] =?UTF-8?q?poi=E4=B8=AD=E8=A1=97=E9=81=93=E4=BF=AE?= =?UTF-8?q?=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../groovy/com/fangjia/script/commData/CommonDatautils.groovy | 2 +- .../groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 3bdde3598..9326a356e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -4679,7 +4679,7 @@ class CommonDatautils { def q = new BasicDBObject() q.put("type","乡镇街道") - q.put("province",new BasicDBObject('$in', ['浙江省','江苏省'])) + q.put("province",new BasicDBObject('$in', ['浙江省','江苏省','山东省'])) def keys = new BasicDBObject() keys.put("province",1) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy index 8edf46edf..76ed1b323 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy @@ -38,7 +38,7 @@ class PoiLocation { task.run(); } def run(){ - def citys = ["莆田市","宁德市","三明市","龙岩市","南平市","漳州市","福州市","厦门市","泉州市","重庆市"] + def citys = ["温州市","嘉兴市","台州市","湖州市","绍兴市","金华市","舟山市","衢州市","丽水市", "南京市","苏州市","无锡市","扬州市","常州市","徐州市","镇江市","南通市","泰州市","连云港市","盐城市","淮安市","宿迁市","济南市","青岛市","烟台市","威海市","潍坊市","淄博市","临沂市","泰安市","菏泽市","东营市","德州市","济宁市","日照市","聊城市","滨州市","枣庄市"] for(def city : citys){ findPois(city) } From 781b8113d7c693837b17275031ac77e4563c6943 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 21 Jun 2024 17:53:32 +0800 Subject: [PATCH 4/9] =?UTF-8?q?=E6=97=A5=E5=BF=97=E5=8F=91=E9=80=81?= =?UTF-8?q?=E6=94=B9=E9=80=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- projects/fangjia-service/pom.xml | 5 +++ .../service/open/OpenUriLogServiceImpl.java | 8 +++- .../util/kafka/KafkaProducerUtils.java | 43 +++++++++++++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 projects/fangjia-service/src/main/java/com/fangjia/service/util/kafka/KafkaProducerUtils.java diff --git a/projects/fangjia-service/pom.xml b/projects/fangjia-service/pom.xml index 0545106d7..2284e8e9c 100644 --- a/projects/fangjia-service/pom.xml +++ b/projects/fangjia-service/pom.xml @@ -234,5 +234,10 @@ okhttp 3.14.2 + + org.apache.kafka + kafka_2.13 + 3.4.0 + diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/open/OpenUriLogServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/open/OpenUriLogServiceImpl.java index a052485eb..cb561d0b6 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/open/OpenUriLogServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/open/OpenUriLogServiceImpl.java @@ -4,6 +4,7 @@ import com.fangjia.dal.QueryParam; import com.fangjia.dal.SortParam; import com.fangjia.dal.mongo.open.dao.OpenUriLogDao; import com.fangjia.dal.mongo.tool.model.OpenUriLog; +import com.fangjia.service.util.kafka.KafkaProducerUtils; import org.apache.commons.lang.StringUtils; import java.util.ArrayList; @@ -13,9 +14,12 @@ import java.util.List; public class OpenUriLogServiceImpl implements OpenUriLogService { private OpenUriLogDao openUriLogDao; + private KafkaProducerUtils kafkaProducerUtils; public void save(OpenUriLog openUriLog) { this.openUriLogDao.save(openUriLog); + // 改造为向Kafka中发送消息 + kafkaProducerUtils.sendLog(openUriLog); } @Override @@ -95,5 +99,7 @@ public class OpenUriLogServiceImpl implements OpenUriLogService { this.openUriLogDao = openUriLogDao; } - + public void setKafkaProducerUtils(KafkaProducerUtils kafkaProducerUtils) { + this.kafkaProducerUtils = kafkaProducerUtils; + } } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/util/kafka/KafkaProducerUtils.java b/projects/fangjia-service/src/main/java/com/fangjia/service/util/kafka/KafkaProducerUtils.java new file mode 100644 index 000000000..83439602a --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/util/kafka/KafkaProducerUtils.java @@ -0,0 +1,43 @@ +package com.fangjia.service.util.kafka; + +import com.alibaba.fastjson.JSONObject; +import com.fangjia.dal.mongo.tool.model.OpenUriLog; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.serialization.StringSerializer; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.context.annotation.Configuration; +import org.springframework.stereotype.Service; + +import java.util.Properties; + +@Configuration +@Service +public class KafkaProducerUtils implements DisposableBean { + public final KafkaProducer producer; + + public KafkaProducerUtils() { + Properties properties = new Properties(); + properties.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.0.100:30092"); + properties.setProperty(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); + properties.setProperty(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); + + this.producer = new KafkaProducer(properties); + } + + public void sendLog(OpenUriLog openUriLog) { + try { + String msg = JSONObject.toJSONString(openUriLog); + ProducerRecord record = new ProducerRecord("log-data-topic", msg); + this.producer.send(record); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + @Override + public void destroy() throws Exception { + producer.close(); + } +} \ No newline at end of file From a982a947938f96eeade06ca719b88014c523d1b8 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 21 Jun 2024 21:42:19 +0800 Subject: [PATCH 5/9] =?UTF-8?q?poi=20=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../DealAndThirdPriceDataProcess.groovy | 2 +- .../ccbevaluation/GuiDingPriceBatch.groovy | 2 +- .../fangjia/script/ccbevaluation/Test.groovy | 17 +++- .../script/commData/CommonDatautils.groovy | 68 +++++++-------- .../script/gj/data_handle/DevProcess.groovy | 34 ++++---- .../script/gj/data_handle/PoiLocation.groovy | 14 ++-- .../gj/data_handle/TownMiddleProcess.groovy | 2 +- .../data_handle/TownshipStrandUpdate.groovy | 84 ++++++++++++++++++- .../fangjia/script/gj/poc/AddPriceTest.groovy | 21 +++-- 9 files changed, 169 insertions(+), 75 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index 5072828fd..023cc157d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -389,7 +389,7 @@ class DealPriceDataProcess { def deleteThirdPartyData(){ // ,"陵水","塔城","保亭","长沙","苏州","南昌","沧州","文昌","衡水" // def citys = ["310000","440100","500000","410100","430100","330100","340100","320100","370200","320500","120000","442000","320300","350200","360100","320400","210200","530100","469005","469028","469029","230700","610700","533400","650400","511100","131100"]//CcbCommonUtils.citys_all - def citys = ["上海","广州"]//CcbCommonUtils.citys_all + def citys = CcbCommonUtils.citys_all def month = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//上个月日期 def rows = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 5c735e4c3..a151ad003 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2024-05-24", "2024-06-21") + def prices = CcbCommonUtils.findPersonPrice(city, "2024-05-24", "2024-06-22") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index d527b08dc..3b8eb91c4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -1289,7 +1289,7 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ -def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240607人工指导价.xlsx" +def fileName = "D:\\数据\\建行\\房产估值\\指导价\\国信达指导价6-21.xlsx" ExcelUtils.multiThreadingModify(fileName, {d-> if(d['J']){ @@ -1301,7 +1301,7 @@ ExcelUtils.multiThreadingModify(fileName, {d-> // d['H'] = dist._id.toString() // } // } - if(!d['A'] && d['J'] instanceof String){ + if(d['J'] instanceof String){ def dist = CommonDatautils.findDistrictById(d['J']) if(dist){ d['A'] = dist.city @@ -1318,6 +1318,19 @@ ExcelUtils.multiThreadingModify(fileName, {d-> },0) +//def file = "D:\\数据\\建行\\房产估值\\指导价\\5城价格翻新6-21.xlsx" +//ExcelUtils.multiThreadingModify(file, { d -> +// if(d['A']){ +// def dist = CommonDatautils.findDistrictById(d['A']) +// if(dist){ +// d['C'] = dist.city +// d['D'] = dist.region +// d['E'] = dist.name +// def random = RandomUtil.random(-300,50) +// d['F'] = random + d['B'] +// } +// } +//}) //def filePath = "D:\\数据\\建行\\房产估值\\小区\\估家独家数据打标签.xlsx" //ExcelUtils.multiThreadingModify(filePath, { d -> // if(d['B']){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 9326a356e..eb1ffe3ab 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1846,39 +1846,6 @@ class CommonDatautils { } return priceMap; } - - - public static def queryDistrictPrice(city, region, name, weekend, cat){ - DBObject queryCondition = new BasicDBObject() - queryCondition.put("city", city); - queryCondition.put("region", region); - queryCondition.put("district_name", name); - queryCondition.put("weekend", weekend); - - if(cat == "district"){ - BasicDBObject queryObj = new BasicDBObject(); - queryObj.putAt(QueryOperators.IN, ["小区","别墅"]); - queryCondition.put("estate_type", queryObj); - }else if(cat == "office"){ - queryCondition.put("estate_type", "写字楼"); - }else if(cat == "shop"){ - queryCondition.put("estate_type", "商铺"); - } - - - DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1)) - cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) - def price = 0; - while(cur.hasNext()) { - DBObject dbo = cur.next() - price = dbo.get("avg_price"); - if(price){ - break; - } - } - - return price; - } public static def queryFirstDistrictPrice(city, region, name){ @@ -1917,6 +1884,39 @@ class CommonDatautils { } + public static def queryDistrictPrice(city, region, name, weekend, cat){ + DBObject queryCondition = new BasicDBObject() + queryCondition.put("city", city); + queryCondition.put("region", region); + queryCondition.put("district_name", name); + queryCondition.put("weekend", weekend); + + if(cat == "district"){ + BasicDBObject queryObj = new BasicDBObject(); + queryObj.putAt(QueryOperators.IN, ["小区","别墅"]); + queryCondition.put("estate_type", queryObj); + }else if(cat == "office"){ + queryCondition.put("estate_type", "写字楼"); + }else if(cat == "shop"){ + queryCondition.put("estate_type", "商铺"); + } + + + DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1)) + cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def price = 0; + while(cur.hasNext()) { + DBObject dbo = cur.next() + price = dbo.get("avg_price"); + if(price){ + break; + } + } + + return price; + } + + public static def queryDistrictPrice(city, region, name){ DBObject queryCondition = new BasicDBObject() queryCondition.put("city", city); @@ -4679,7 +4679,7 @@ class CommonDatautils { def q = new BasicDBObject() q.put("type","乡镇街道") - q.put("province",new BasicDBObject('$in', ['浙江省','江苏省','山东省'])) +// q.put("province",new BasicDBObject('$in', ['浙江省','江苏省','山东省'])) def keys = new BasicDBObject() keys.put("province",1) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index 4dbf6a57c..de279cbea 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -39,7 +39,7 @@ class DevProcess { // task.calAddrDistince() // task.updateCityAndDist() - task.addDevAddres1() //按添加地址中包含开发区名称的地址 +// task.addDevAddres1() //按添加地址中包含开发区名称的地址 // task.addDevAddres2()//添加相差2公里范围内的地址 // task.matchDevAndAddAddress() @@ -48,7 +48,7 @@ class DevProcess { // task.createDevMapping() // task.deleteAddr() // task.deleteAddr2() -// task.synchDevTo43() + task.synchDevTo43() // task.addNewDev2() // task.addAlias() @@ -644,16 +644,16 @@ class DevProcess { def matchDevAndAddAddress(){ - ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\新跑省份\\福建去重地址入库.xlsx",{d-> - if(d['J']){ + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\新跑省份\\福建重庆去重后地址.xlsx",{d-> + if(d['E']){ def q = new BasicDBObject() - q.put("name", d['J']) + q.put("name", d['E']) def obj = development_zone_CONN.findOne(q) if(obj){ - d['P'] = obj._id.toString() - d['Q'] = obj.name - def address = d['L']+d['M'] + d['F'] = obj._id.toString() + d['G'] = obj.name + def address = d['D'] q.put("address", address) // def street = d['G'] // q.put("township", street) @@ -685,14 +685,14 @@ class DevProcess { def matchDevTownship(){ - ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\区域匹配开发区.xlsx",{d-> - if(d['C']){ + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\新跑省份\\福建重庆去重后地址.xlsx",{d-> + if(d['D']){ def q = new BasicDBObject() - q.put("name", d['C']) + q.put("name", d['D']) def obj = development_zone_CONN.findOne(q) if(obj){ - d['E'] = obj._id.toString() - def street = d['D'] + d['F'] = obj._id.toString() + def street = d['E'] q.put("township", street) def sameObj = DEV_MAPPING_CONN.findOne(q) if(!sameObj){ @@ -706,11 +706,11 @@ class DevProcess { } udata.put("name",obj.name) -// udata.put("type", "乡镇街道") -// udata.put("township", street) + udata.put("type", "乡镇街道") + udata.put("township", street) - udata.put("type", "区域") - udata.put("dev_district", street) +// udata.put("type", "区域") +// udata.put("dev_district", street) udata.put("c_date",new Date()) DEV_MAPPING_CONN.save(udata) println obj.name +" "+street diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy index 76ed1b323..8f2333705 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy @@ -39,15 +39,15 @@ class PoiLocation { } def run(){ def citys = ["温州市","嘉兴市","台州市","湖州市","绍兴市","金华市","舟山市","衢州市","丽水市", "南京市","苏州市","无锡市","扬州市","常州市","徐州市","镇江市","南通市","泰州市","连云港市","盐城市","淮安市","宿迁市","济南市","青岛市","烟台市","威海市","潍坊市","淄博市","临沂市","泰安市","菏泽市","东营市","德州市","济宁市","日照市","聊城市","滨州市","枣庄市"] - for(def city : citys){ - findPois(city) - } +// for(def city : citys){ + findPois() +// } } - def findPois(def city){ + def findPois(){ def q = new BasicDBObject() - q.put("city", city) +// q.put("city", city) q.put("run_status", new BasicDBObject('$exists', false)) def keys = new BasicDBObject() @@ -115,7 +115,9 @@ class PoiLocation { def regionFlag = null if(d.district){ for(def rs : regions){ - regionFlag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(rs.shape.coordinates)) + if(rs.shape && rs.shape.coordinates){ + regionFlag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(rs.shape.coordinates)) + } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy index f3d00c82c..540fc1ffa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy @@ -50,7 +50,7 @@ class TownMiddleProcess { //修改乡镇街道别名 // updateTownAlias() //中间表同步到100 -// synchTown() + synchTown() //添加地级市标志 // addCityFlag() // addPinyin() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy index f6e830017..12012bc8e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy @@ -3,9 +3,11 @@ package com.fangjia.script.gj.data_handle import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.looplineinfo.InsidePolygon import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.BasicDBObject import com.mongodb.DBCollection import java.util.concurrent.CopyOnWriteArrayList @@ -27,7 +29,9 @@ class TownshipStrandUpdate { static main(args) { def task = new TownshipStrandUpdate() - task.process() +// task.process() + task.updateTown() +// task.stat() } @@ -42,13 +46,14 @@ class TownshipStrandUpdate { def citys = CommonDatautils.findAllCity() citys = citys.reverse() for(def city : citys){ - if(["中山"].contains(city)){ +// + if(["中山","东莞"].contains(city)){ continue } - def fileName = "D:\\数据\\数据建设\\乡镇街道\\小区\\小区乡镇街道格式化-${city}.xlsx" + def fileName = "D:\\数据\\数据建设\\乡镇街道\\商铺\\商铺乡镇街道格式化-${city}.xlsx" File file=new File(fileName); if(!file.exists()){ - def dists = CommonDatautils.findAllDistrictsFromMg(city, "district", ["city","region","name","address","cat","estate_type2","status","township","lng","lat"]) + def dists = CommonDatautils.findAllDistrictsFromMg(city, "shop", ["city","region","name","address","cat","estate_type2","status","township","lng","lat"]) dists = checkData(dists) def datas = createDatas(dists) ExcelUtils.write(fileName, datas, "${city}") @@ -56,6 +61,77 @@ class TownshipStrandUpdate { } } +// 相同且为统计局乡镇 +// 更正 +// 不同且不为统计局乡镇 +// 区域不同 +// 地图无乡镇街道 +// 城市不同 + + def stat(){ + def citys = CommonDatautils.findAllCity() + citys = citys.reverse() + def rets = [] + rets << ["城市", "小区量", "更正量","相同且为统计局乡镇","不同且不为统计局乡镇","区域不同","地图无乡镇街道","城市不同","无值"] + for(def city : citys){ + def fileName = "D:\\数据\\数据建设\\乡镇街道\\商铺\\商铺乡镇街道格式化-${city}.xlsx" + File file=new File(fileName); + def map = [:] + def datas = null + if(file.exists()){ + datas = ExcelUtils.read(fileName,0, 1) + for(def d : datas){ + def key = "空值" + if(d['S']){ + key = d['S'] + } + def count = map.get(key) + if(!count){ + count = 0 + } + map.put(key, count+1) + } + } + + println city + map.each {k,v-> + println k+" "+v + } + println "--------------------------------------------------------------------------------------------" + if(map){ + rets << [city, datas.size(), map.get("更正"),map.get("相同且为统计局乡镇"),map.get("不同且不为统计局乡镇"),map.get("区域不同"),map.get("地图无乡镇街道"),map.get("城市不同"),map.get("空值")] + } + + } + ExcelUtils.write("D:\\数据\\数据建设\\乡镇街道\\乡镇街道更新结果统计.xlsx", rets, "商铺") + } + + def updateTown(){ + def citys = CommonDatautils.findAllCity() + citys = citys.reverse() + for(def city : citys){ + def fileName = "D:\\数据\\数据建设\\乡镇街道\\小区\\小区乡镇街道格式化-${city}.xlsx" + File file=new File(fileName); + if(file.exists()){ + println fileName + ExcelUtils.multiThreadingModify(fileName,{d-> + if(d['S']!='相同且为统计局乡镇' && d['S']){ + def q = new BasicDBObject() + q.put("city", d['A']) + q.put("region", d['B']) + q.put("name", d['C']) + q.put("status", 0) + q.put("cat","district") + + def udata = new BasicDBObject() + udata.put("township", d['R']) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + } + }) + } + } + } + def checkData(def dists){ def totalCount = new AtomicLong() def rets = new CopyOnWriteArrayList() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy index f0c4282e1..025a68f58 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy @@ -33,11 +33,11 @@ class AddPriceTest { task.run(); } def run(){ - def filePath = "D:\\数据\\建行\\房产估值\\神经网络计算.xlsx" - addGXDPrice(filePath) + def filePath = "D:\\数据\\建行\\房产估值\\0621.xlsx" +// addGXDPrice(filePath) // addCcbPrice(filePath) // addThirdPrice(filePath) -// addFjwPrice(filePath) + addFjwPrice(filePath) // exportPkPrice() // addRegion() // addTradePrice(filePath) @@ -205,7 +205,7 @@ class AddPriceTest { def addThirdPrice(filepath){ ExcelUtils.multiThreadingModify(filepath, { d -> if (!d['X']) { - def dist = CommonDatautils.findDistrictById(d['Y']) + def dist = CommonDatautils.findDistrictById(d['A']) if(dist){ def thirdPrice = findThirdPartyPrice(dist.city, dist.region, dist.name) d['X'] = thirdPrice @@ -220,8 +220,8 @@ class AddPriceTest { q.put("city", city) q.put("region", region) q.put("name", name) -// q.put("s_date", new BasicDBObject('$gte', 202311).append('$lte',202404)) - q.put("s_date", 202404) + q.put("s_date", new BasicDBObject('$gte', 202403)) +// q.put("s_date", 202404) def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1)) def price = null for(def db : dbcursor){ @@ -271,13 +271,16 @@ class AddPriceTest { def addFjwPrice(path){ ExcelUtils.multiThreadingModify(path,{d-> if(!d['L']){ - if(d['B'] instanceof String){ - def dist = CommonDatautils.findDistrictById(d['B']) + if(d['A'] instanceof String){ + def dist = CommonDatautils.findDistrictById(d['A']) if(dist){ + d['A'] = dist.city + d['B'] = dist.region + d['C'] = dist.name // d['N'] = dist.region def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name) if(dbo){ -// d['W'] = dbo.avg_price + d['W'] = dbo.avg_price // d['X'] = dbo.cal_type d['L'] = dbo.avg_price_2b } From 0fbfe17c6dfa40de0b81939163f36e73b018fa09 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sat, 22 Jun 2024 00:09:40 +0800 Subject: [PATCH 6/9] =?UTF-8?q?=E5=9F=8E=E5=B8=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/gj/data_handle/PoiLocation.groovy | 12 ++++++------ .../gj/data_handle/TownshipStrandUpdate.groovy | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy index 8f2333705..1e29bec47 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy @@ -38,16 +38,16 @@ class PoiLocation { task.run(); } def run(){ - def citys = ["温州市","嘉兴市","台州市","湖州市","绍兴市","金华市","舟山市","衢州市","丽水市", "南京市","苏州市","无锡市","扬州市","常州市","徐州市","镇江市","南通市","泰州市","连云港市","盐城市","淮安市","宿迁市","济南市","青岛市","烟台市","威海市","潍坊市","淄博市","临沂市","泰安市","菏泽市","东营市","德州市","济宁市","日照市","聊城市","滨州市","枣庄市"] -// for(def city : citys){ - findPois() -// } + def citys = ["重庆市","杭州市","泉州市","东莞市","福州市","厦门市","佛山市","天津市","武汉市","宁波市","郑州市","长沙市","青岛市","上海市","西安市","温州市","无锡市","南京市","湖州市","合肥市","临沂市","石家庄市","昆明市","金华市","保定市","成都市","济南市","北京市","沈阳市","潍坊市","广州市","惠州市","嘉兴市","赣州市","苏州市","南通市","南宁市","台州市","南阳市","徐州市","长春市","深圳市","哈尔滨市","漳州市","中山市","常州市","烟台市","大连市","邯郸市","济宁市","盐城市","绍兴市","唐山市","洛阳市","阜阳市","菏泽市","贵阳市","沧州市","遵义市","莆田市","南昌市","商丘市","周口市","邢台市","江门市","衡阳市","邵阳市","信阳市","廊坊市","扬州市","泰州市","新乡市","太原市","德州市","淄博市","宁德市","绵阳市","汕头市","聊城市","淮安市","驻马店市","黄冈市","宿迁市","三明市","六安市","龙岩市","桂林市","岳阳市","南平市","揭阳市","上饶市","九江市","南充市","连云港市","襄阳市","泰安市","常德市","郴州市","安阳市","安庆市","湛江市","永州市","茂名市","曲靖市","宜宾市","肇庆市","宿州市","兰州市","运城市","镇江市","滁州市","威海市","梅州市","宜春市","清远市","亳州市","怀化市","荆州市","平顶山市","开封市","玉林市","咸阳市","宜昌市","株洲市","乌鲁木齐市","滨州市","渭南市","临汾市","榆林市","芜湖市","银川市","衡水市","毕节市","枣庄市","吉安市","张家口市","柳州市","许昌市","泸州市","呼和浩特市","达州市","德阳市","赤峰市","珠海市","孝感市","益阳市","娄底市","乐山市","恩施土家族苗族自治州","焦作市","湘潭市","海口市","红河哈尼族彝族自治州","黔南布依族苗族自治州","河源市","汉中市","日照市","大理白族自治州","濮阳市","黔东南苗族侗族自治州","秦皇岛市","晋中市","宝鸡市","吉林市","包头市","蚌埠市","东营市","宣城市","韶关市","衢州市","十堰市","承德市","丽水市","眉山市","鞍山市","昭通市","凉山彝族自治州","长治市","抚州市","鄂尔多斯市","吕梁市","百色市","齐齐哈尔市","贵港市","潮州市","黔西南布依族苗族自治州","广安市","淮南市","河池市","大同市","铜仁市","阳江市","咸宁市","营口市","绥化市","马鞍山市","安康市","荆门市","忻州市","广元市","内江市","遂宁市","西宁市","大庆市","汕尾市","文山壮族苗族自治州","锦州市","延安市","巴中市","呼伦贝尔市","楚雄彝族自治州","黄石市","玉溪市","六盘水市","通辽市","钦州市","云浮市","梧州市","漯河市","晋城市","庆阳市","安顺市","丹东市","自贡市","朝阳市","葫芦岛市","天水市","三门峡市","湘西土家族苗族自治州","北海市","陇南市","普洱市","延边朝鲜族自治州","保山市","定西市","随州市","喀什地区","黄山市","佳木斯市","巴彦淖尔市","牡丹江市","舟山市","资阳市","商洛市","松原市","锡林郭勒盟","伊犁哈萨克自治州","萍乡市","铁岭市","乌兰察布市","丽江市","景德镇市","张家界市","抚顺市","通化市","淮北市","来宾市","临沧市","阿克苏地区","贺州市","盘锦市","昌吉回族自治州","池州市","雅安市","三亚市","平凉市","四平市","酒泉市","西双版纳傣族自治州","鹤壁市","辽阳市","白城市","吴忠市","崇左市","阜新市","朔州市","白银市","拉萨市","巴音郭楞蒙古自治州","张掖市","铜陵市","武威市","临夏回族自治州","德宏傣族景颇族自治州","新余市","甘孜藏族自治州","兴安盟","本溪市","鄂州市","黑河市","阿坝藏族羌族自治州","阳泉市","防城港市","鸡西市","仙桃市","鹰潭市","固原市","白山市","中卫市","双鸭山市","攀枝花市","石嘴山市","和田地区","海东市","塔城地区","日喀则市","济源市","伊春市","天门市","辽源市","潜江市","那曲市","铜川市","海西蒙古族藏族自治州","鹤岗市","昌都市","阿拉善盟","乌海市","儋州市","哈密市","克拉玛依市","七台河市","金昌市","甘南藏族自治州","琼海市","迪庆藏族自治州","石河子市","阿勒泰地区","文昌市","玉树藏族自治州","嘉峪关市","万宁市","怒江傈僳族自治州","林芝市","海南藏族自治州","吐鲁番市","澄迈县","陵水黎族自治县","博尔塔拉蒙古自治州","山南市","阿里地区","大兴安岭地区","海北藏族自治州","东方市","乐东黎族自治县","克孜勒苏柯尔克孜自治州","果洛藏族自治州","临高县","阿拉尔市","黄南藏族自治州","神农架林区","屯昌县","定安县","昌江黎族自治县","五家渠市","保亭黎族苗族自治县","琼中黎族苗族自治县","五指山市","白沙黎族自治县","北屯市","图木舒克市","可克达拉市","双河市","铁门关市","毕节地区","昆玉市","铜仁地区","海东地区","三沙市","吐鲁番地区","哈密地区","日喀则地区","林芝地区","昌都地区","山南地区","那曲地区"] + for(def city : citys){ + findPois(city) + } } - def findPois(){ + def findPois(city){ def q = new BasicDBObject() -// q.put("city", city) + q.put("city", city) q.put("run_status", new BasicDBObject('$exists', false)) def keys = new BasicDBObject() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy index 12012bc8e..c4816c69c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy @@ -110,7 +110,7 @@ class TownshipStrandUpdate { def citys = CommonDatautils.findAllCity() citys = citys.reverse() for(def city : citys){ - def fileName = "D:\\数据\\数据建设\\乡镇街道\\小区\\小区乡镇街道格式化-${city}.xlsx" + def fileName = "D:\\数据\\数据建设\\乡镇街道\\商铺\\商铺乡镇街道格式化-${city}.xlsx" File file=new File(fileName); if(file.exists()){ println fileName @@ -121,7 +121,7 @@ class TownshipStrandUpdate { q.put("region", d['B']) q.put("name", d['C']) q.put("status", 0) - q.put("cat","district") + q.put("cat","shop") def udata = new BasicDBObject() udata.put("township", d['R']) From e0acc767a717d2ac9648bffb7667ffec8dfedaab Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sat, 22 Jun 2024 00:33:55 +0800 Subject: [PATCH 7/9] up --- .../groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy index 1e29bec47..32eb9ca85 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy @@ -48,7 +48,7 @@ class PoiLocation { def findPois(city){ def q = new BasicDBObject() q.put("city", city) - q.put("run_status", new BasicDBObject('$exists', false)) + q.put("run_status", new BasicDBObject('$ne', 1)) def keys = new BasicDBObject() keys.put("_id", 1) From 12080cb162a3b9429ddcacac5c7f00f969076d85 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sat, 22 Jun 2024 10:20:00 +0800 Subject: [PATCH 8/9] up --- .../fangjia/script/gj/data_handle/PoiLocation.groovy | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy index 32eb9ca85..45520fd29 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy @@ -231,9 +231,14 @@ class PoiLocation { } """ HttpRequest request = new HttpRequest() - String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8") - def response = JSONObject.fromObject(res) - println(response) + try{ + String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8") + def response = JSONObject.fromObject(res) + println(response) + }catch(Exception e){ + println "error : "+e.getMessage() + } + } From 31edd0140e93e8330ef57a5772f2bbb36df556d8 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sat, 22 Jun 2024 10:25:14 +0800 Subject: [PATCH 9/9] up --- .../script/gj/data_handle/PoiLocation.groovy | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy index 45520fd29..b83df941e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy @@ -67,8 +67,8 @@ class PoiLocation { index++ def datas = checkData(index, rets) // updateToEs(datas) - updateToEs2(datas) - updateToMG(datas) +// updateToEs2(datas) +// updateToMG(datas) rets = [] print "." } @@ -92,6 +92,10 @@ class PoiLocation { if(map.update_status && map.update_status!=100){ println "[ ${index} ]"+d.city +" "+totalCount.get() +"/"+rets.size()+" "+map.id+" "+d.district+" 原:"+d.township+" 新:"+map.township } + if (map.update_status && map.update_status !=100) { + updateMapPoi(map.id, map.township) + } + updateToMG(map) } catch (Exception e) { e.printStackTrace() println d.city +" "+d.name @@ -300,10 +304,10 @@ class PoiLocation { } - def updateToMG(rows){ - def rets = [] - for(def i=0;i if(k!="id"){ @@ -318,7 +322,7 @@ class PoiLocation { println m.toString()+" 无法修改-无值" } - } +// } } }