From a30b1604b143ffdf24e97b271c4659fe8cb07782 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Wed, 21 Aug 2024 20:46:42 +0800 Subject: [PATCH 1/3] update --- .../main/java/com/fangjia/core/Constants.java | 2 +- .../script/ccbevaluation/AddSzDealData.groovy | 2 +- .../script/gj/data_handle/DevProcess.groovy | 2 +- .../script/gj/data_handle/MinglujiData.groovy | 41 +++++++++++++++++-- .../script/gj/village/QiliangTest.groovy | 2 +- .../gj/xiamen/GaoDePoiDataProcess.groovy | 6 +-- .../com/fangjia/script/util/MongoUtils.groovy | 4 ++ 7 files changed, 49 insertions(+), 10 deletions(-) diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/Constants.java b/projects/fangjia-core/src/main/java/com/fangjia/core/Constants.java index 7a9f45104..2cc2c080c 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/Constants.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/Constants.java @@ -7,7 +7,7 @@ public interface Constants { String FANGJIA_CRAWL_MONGO_URI = "mongodb://192.168.0.235:27018"; String LOG_MONGO_URI = "mongodb://192.168.0.42:52002"; String SEABIRD_MONGO_URI = "mongodb://192.168.0.74:57007"; - String SEAWATER_MONGO_URI = "mongodb://192.168.0.206:27020"; + String SEAWATER_MONGO_URI = "mongodb://root:Goojia%40206!!#@192.168.0.206:27020"; String MSVC_FANG_JIA_DATA_SERVICE_BASE_PATH ="http://192.168.0.100:38080/FangJiaDataService"; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy index 449e1ad79..974df17ee 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy @@ -9,7 +9,7 @@ import com.mongodb.DBCollection /** * 深圳公众号成交数据录入 */ -def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年6月深圳二手房成交.xlsx", 0, 1) +def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年7月深圳二手房成交.xlsx", 0, 1) def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index 411c7e92e..861999ae8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -33,7 +33,7 @@ class DevProcess { DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com") DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com") - DBCollection kfq_COLL = MongoUtils.getCollectionByName("crawl", "kfq_investchn_com", "192.168.0.206",27020) +// DBCollection kfq_COLL = MongoUtils.getCollectionByName("crawl", "kfq_investchn_com", "192.168.0.206",27020) static main(args) { def task = new DevProcess(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy index 327e48a2e..3f489e8cf 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy @@ -25,9 +25,11 @@ class MinglujiData { } def run(){ -// exportDatas("湖州") + exportDatas("嘉兴") // matchDev("浙江省","湖州") - matchCommunity("湖州") +// matchCommunity("湖州") + +// tempMatchDev("浙江省") } def exportDatas(city){ def datas = findMinglujiDatas(city) @@ -66,8 +68,41 @@ class MinglujiData { }) } + def tempMatchDev(prov){ + def devs = loadAllDevs(prov) + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\湖州-开发区-等级.xlsx",{d-> + if(d['U'] && !d['W'] ){ + def obj = findDevByName(d['U']) + if(!obj){ + obj = findDevByAlias(d['U']) + if(!obj){ + obj = findDebBySimlar(devs, d['U'], d['B']) + if(!obj){ + obj = findDevByPark(d['A'], d['B'], d['U']) + if (!obj) { + }else{ + d['X'] = "园区" + } + }else{ + d['X'] = "相似" + } + }else{ + d['X'] = "别名" + } + }else{ + d['X'] = "相同" + } + if(obj){ + d['W'] = obj.name + d['Y'] = obj.city + d['Z'] = obj.district + } + } + }) + } + def matchCommunity(city){ - ExcelUtils.modify("D:\\数据\\数据建设\\名录集\\${city}-没配居委1.xlsx",{d-> + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\${city}-没配居委1.xlsx",{d-> if(d['A'] && d['B'] && d['L'] && !d['Q']){ def list = findCommunityByName(d['A'], d['B'], d['L']) if(!list){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy index c2f11e6f8..527a01dea 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy @@ -27,7 +27,7 @@ class QiliangTest { // task.run() def data = [ - 'address': '景县开发区东区', + 'address': '湖州市南太湖高新技术产业园区大港村陈家斗80号', 'type': '1' //1:对公 0:对私 ] def s = task.getProxyResult(data); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy index b8d657c89..0a4f07a1c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy @@ -126,8 +126,7 @@ class GaoDePoiDataProcess extends BaseBatch{ } def sysnchVillage(){ - def buducitys = [ '七台河市', '上海市','万宁市', '三亚市', '三明市', '三门峡市', '上饶市', '东方市', '东莞市', - '东营市', '中卫市', '中山市', '临夏回族自治州', '临汾市', '临沂市', '临沧市', '临高县', '丹东市', + def buducitys = [ '临汾市', '临沂市', '临沧市', '临高县', '丹东市', '丽水市', '丽江市', '乌兰察布市', '乌海市', '乌鲁木齐市', '乐东黎族自治县', '乐山市', '九江市', '云浮市', '五家渠市', '五指山市', '亳州市', '仙桃市', '伊春市', '伊犁哈萨克自治州', '佛山市', '佳木斯市', '保亭黎族苗族自治县', '保定市', '保山市', '信阳市', '儋州市', '克孜勒苏柯尔克孜自治州', @@ -169,7 +168,8 @@ class GaoDePoiDataProcess extends BaseBatch{ '黔东南苗族侗族自治州', '黔南布依族苗族自治州', '黔西南布依族苗族自治州', '齐齐哈尔市', '龙岩市'] //"丽水市","杭州市", - def runbuducitys = ["湖州市","宁波市","温州市","嘉兴市","台州市","绍兴市","金华市","舟山市","衢州市"] + def runbuducitys = ["湖州市","宁波市","温州市","嘉兴市","台州市","绍兴市","金华市","舟山市","衢州市",'七台河市', '上海市','万宁市', '三亚市', '三明市', '三门峡市', '上饶市', '东方市', '东莞市', + '东营市', '中卫市', '中山市', '临夏回族自治州'] for(def city : buducitys){ if(runbuducitys.contains(city)){ continue diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/MongoUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/MongoUtils.groovy index ec6fa0a0b..54d093d7e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/MongoUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/MongoUtils.groovy @@ -71,6 +71,10 @@ class MongoUtils { return com.fangjia.core.util.MongoUtils.getDbCollection(Constants.TEST_FANGJIA_MONGO_URI, db, name) } + if ((domainUrl == "192.168.0.206") && (port == 27020)) { + return com.fangjia.core.util.MongoUtils.getDbCollection(Constants.SEAWATER_MONGO_URI, db, name) + } + def key = db + name + domainUrl + port if (collectionMap.containsKey(key)) { return collectionMap[key] From 6d9d4b061a48fa0f7e394b9eb0b6eeb835bb9001 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Thu, 22 Aug 2024 12:18:43 +0800 Subject: [PATCH 2/3] update --- .../gj/data_handle/DevUpdateTestBatch.groovy | 5 +- .../script/gj/data_handle/MinglujiData.groovy | 179 +++++++++++++----- 2 files changed, 132 insertions(+), 52 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy index 052f2d046..84d14b8e0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy @@ -31,9 +31,12 @@ class DevUpdateTestBatch { def task = new DevUpdateTestBatch() def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx // task.addDev(path) - task.addDevAddress(path) +// task.addDevAddress(path) // task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是") + + + } def addDev(path){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy index 3f489e8cf..3ced98380 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy @@ -1,9 +1,11 @@ package com.fangjia.script.gj.data_handle +import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import com.rabbitmq.client.impl.AMQImpl import java.util.regex.Pattern @@ -19,17 +21,25 @@ class MinglujiData { DBCollection dev_mapping_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL) DBCollection community_100_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL) + + DBCollection company_100_CONN = MongoUtils.getCollectionByName2("fangjia", "company", MONGO100_URL) + static main(args) { def task = new MinglujiData() task.run() } def run(){ - exportDatas("嘉兴") -// matchDev("浙江省","湖州") -// matchCommunity("湖州") + def prov = "浙江省" + def city = "嘉兴" +// exportDatas(city) +// matchDev(prov, city) + matchCommunity(city) -// tempMatchDev("浙江省") +// tempMatchDev(prov) + + +// saveMljData() } def exportDatas(city){ def datas = findMinglujiDatas(city) @@ -37,35 +47,70 @@ class MinglujiData { } def matchDev(prov, city){ def devs = loadAllDevs(prov) - ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\${city}.xlsx",{d-> - if(d['E'] && !d['N'] ){ - def obj = findDevByName(d['E']) - if(!obj){ - obj = findDevByAlias(d['E']) + + def i = 1; + while(i<20) { + def fileName = "/home/goojia/gj/mlj_${i}.xlsx" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\数据建设\\名录集\\${city}_${i}.xlsx" + } + ExcelUtils.multiThreadingModify(fileName,{d-> + if(d['D']){ + def street = d['D'] + def splitStreet = "" + if(street){ + if(street.contains(d['C'])){ + def start = street.toString().lastIndexOf(d['C'])+d['C'].size() + def end = street.toString().size() + splitStreet = street.toString().substring(start, end) + }else{ + splitStreet = street + } + } + d['K'] = splitStreet + } + + if(d['I']){ + def splitCommunity = "" + if(d['C'] && d['I'].toString().contains(d['C'])){ + splitCommunity = d['I'].toString().substring(d['I'].toString().lastIndexOf(d['C'])+d['C'].toString().size(), d['I'].toString().size()) + }else{ + splitCommunity = d['I'] + } + d['L'] = splitCommunity + } + + + if(d['E'] && !d['N'] ){ + def obj = findDevByName(d['E']) if(!obj){ - obj = findDebBySimlar(devs, d['E'], d['B']) + obj = findDevByAlias(d['E']) if(!obj){ + obj = findDebBySimlar(devs, d['E'], d['B']) + if(!obj){ obj = findDevByPark(d['A'], d['B'], d['E']) if (!obj) { }else{ d['N'] = "园区" } + }else{ + d['N'] = "相似" + } }else{ - d['N'] = "相似" + d['N'] = "别名" } }else{ - d['N'] = "别名" + d['N'] = "相同" + } + if(obj){ + d['M'] = obj.name + d['O'] = obj.city + d['P'] = obj.district } - }else{ - d['N'] = "相同" } - if(obj){ - d['M'] = obj.name - d['O'] = obj.city - d['P'] = obj.district - } - } - }) + }) + } + } def tempMatchDev(prov){ @@ -102,39 +147,71 @@ class MinglujiData { } def matchCommunity(city){ - ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\${city}-没配居委1.xlsx",{d-> - if(d['A'] && d['B'] && d['L'] && !d['Q']){ - def list = findCommunityByName(d['A'], d['B'], d['L']) - if(!list){ - list = findCommunityBySimlar(d['A'], d['B'], d['L']) - if(!list){ + def i = 0; + while(i<20){ + def fileName = "/home/goojia/gj/mlj_${i}.xlsx" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\数据建设\\名录集\\${city}_${i}.xlsx" + } - }else{ - d['T'] = "包含" - } - }else{ - d['T'] = "相同" - } - if(list){ - if(list.size()==1){ - def obj = list[0] - d['Q'] = obj.name - d['R'] = obj.street_name - }else if(list.size()>=2){ - for(def l : list){ - if(d['C']==l.area_name){ - d['Q'] = l.name - d['R'] = l.street_name - break; - } - } - d['S'] = list.size() - } - } - } - }, 0) + File file=new File(fileName); + if(file.exists()){ + ExcelUtils.multiThreadingModify(fileName,{d-> + if(d['A'] && d['B'] && d['L'] && !d['Q']){ + def list = findCommunityByName(d['A'], d['B'], d['L']) + if(!list){ + list = findCommunityBySimlar(d['A'], d['B'], d['L']) + if(!list){ + + }else{ + d['T'] = "包含" + } + }else{ + d['T'] = "相同" + } + if(list){ + if(list.size()==1){ + def obj = list[0] + d['Q'] = obj.name + d['R'] = obj.street_name + d['U'] = obj.area_name + d['V'] = obj.city_name + }else if(list.size()>=2){ + for(def l : list){ + if(d['C']==l.area_name){ + d['Q'] = l.name + d['R'] = l.street_name + d['U'] = l.area_name + d['V'] = l.city_name + break; + } + } + d['S'] = list.size() + } + } + } + }, 0) + println fileName +" 远行完成" + } + + } } + + def saveMljData(){ + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\湖州入库数据.xlsx",{d-> + def obj = new BasicDBObject() + obj.put("province", d['A']) + obj.put("city", d['B']) + obj.put("district", d['C']) + obj.put("name", d['D']) + obj.put("address", d['E']) + obj.put("dev_name", d['F']) + obj.put("committee", d['G']) + obj.put("street", d['H']) + company_100_CONN.save(obj) + }) + } def loadAllDevs(prov){ def q = new BasicDBObject() q.put("province", prov) From 63e6e0ff9f9081a0ded56a209d0702bf296a4e98 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Thu, 22 Aug 2024 12:43:17 +0800 Subject: [PATCH 3/3] update --- .../script/gj/data_handle/MinglujiData.groovy | 113 +++++++++--------- 1 file changed, 59 insertions(+), 54 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy index 3ced98380..538bc0a44 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy @@ -54,61 +54,66 @@ class MinglujiData { if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\数据建设\\名录集\\${city}_${i}.xlsx" } - ExcelUtils.multiThreadingModify(fileName,{d-> - if(d['D']){ - def street = d['D'] - def splitStreet = "" - if(street){ - if(street.contains(d['C'])){ - def start = street.toString().lastIndexOf(d['C'])+d['C'].size() - def end = street.toString().size() - splitStreet = street.toString().substring(start, end) - }else{ - splitStreet = street - } - } - d['K'] = splitStreet - } - - if(d['I']){ - def splitCommunity = "" - if(d['C'] && d['I'].toString().contains(d['C'])){ - splitCommunity = d['I'].toString().substring(d['I'].toString().lastIndexOf(d['C'])+d['C'].toString().size(), d['I'].toString().size()) - }else{ - splitCommunity = d['I'] - } - d['L'] = splitCommunity - } - - - if(d['E'] && !d['N'] ){ - def obj = findDevByName(d['E']) - if(!obj){ - obj = findDevByAlias(d['E']) - if(!obj){ - obj = findDebBySimlar(devs, d['E'], d['B']) - if(!obj){ - obj = findDevByPark(d['A'], d['B'], d['E']) - if (!obj) { - }else{ - d['N'] = "园区" + File file=new File(fileName); + if(file.exists()){ + ExcelUtils.multiThreadingModify(fileName, { d -> + if (d['D']) { + def street = d['D'] + def splitStreet = "" + if (street) { + if (street.contains(d['C'])) { + def start = street.toString().lastIndexOf(d['C']) + d['C'].size() + def end = street.toString().size() + splitStreet = street.toString().substring(start, end) + } else { + splitStreet = street } - }else{ - d['N'] = "相似" } - }else{ - d['N'] = "别名" + d['K'] = splitStreet } - }else{ - d['N'] = "相同" - } - if(obj){ - d['M'] = obj.name - d['O'] = obj.city - d['P'] = obj.district - } - } - }) + + if (d['I']) { + def splitCommunity = "" + if (d['C'] && d['I'].toString().contains(d['C'])) { + splitCommunity = d['I'].toString().substring(d['I'].toString().lastIndexOf(d['C']) + d['C'].toString().size(), d['I'].toString().size()) + } else { + splitCommunity = d['I'] + } + d['L'] = splitCommunity + } + + + if (d['E'] && !d['N']) { + def obj = findDevByName(d['E']) + if (!obj) { + obj = findDevByAlias(d['E']) + if (!obj) { + obj = findDebBySimlar(devs, d['E'], d['B']) + if (!obj) { + obj = findDevByPark(d['A'], d['B'], d['E']) + if (!obj) { + } else { + d['N'] = "园区" + } + } else { + d['N'] = "相似" + } + } else { + d['N'] = "别名" + } + } else { + d['N'] = "相同" + } + if (obj) { + d['M'] = obj.name + d['O'] = obj.city + d['P'] = obj.district + } + } + }) + } + i++ + println fileName +" 完成" } } @@ -147,7 +152,7 @@ class MinglujiData { } def matchCommunity(city){ - def i = 0; + def i = 1; while(i<20){ def fileName = "/home/goojia/gj/mlj_${i}.xlsx" if(CommonDatautils.isWindow()){ @@ -193,7 +198,7 @@ class MinglujiData { }, 0) println fileName +" 远行完成" } - + i++ } }