diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/Constants.java b/projects/fangjia-core/src/main/java/com/fangjia/core/Constants.java index 7a9f45104..2cc2c080c 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/Constants.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/Constants.java @@ -7,7 +7,7 @@ public interface Constants { String FANGJIA_CRAWL_MONGO_URI = "mongodb://192.168.0.235:27018"; String LOG_MONGO_URI = "mongodb://192.168.0.42:52002"; String SEABIRD_MONGO_URI = "mongodb://192.168.0.74:57007"; - String SEAWATER_MONGO_URI = "mongodb://192.168.0.206:27020"; + String SEAWATER_MONGO_URI = "mongodb://root:Goojia%40206!!#@192.168.0.206:27020"; String MSVC_FANG_JIA_DATA_SERVICE_BASE_PATH ="http://192.168.0.100:38080/FangJiaDataService"; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy index 449e1ad79..974df17ee 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy @@ -9,7 +9,7 @@ import com.mongodb.DBCollection /** * 深圳公众号成交数据录入 */ -def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年6月深圳二手房成交.xlsx", 0, 1) +def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年7月深圳二手房成交.xlsx", 0, 1) def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index 411c7e92e..861999ae8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -33,7 +33,7 @@ class DevProcess { DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com") DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com") - DBCollection kfq_COLL = MongoUtils.getCollectionByName("crawl", "kfq_investchn_com", "192.168.0.206",27020) +// DBCollection kfq_COLL = MongoUtils.getCollectionByName("crawl", "kfq_investchn_com", "192.168.0.206",27020) static main(args) { def task = new DevProcess(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy index 052f2d046..84d14b8e0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy @@ -31,9 +31,12 @@ class DevUpdateTestBatch { def task = new DevUpdateTestBatch() def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx // task.addDev(path) - task.addDevAddress(path) +// task.addDevAddress(path) // task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是") + + + } def addDev(path){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy index 327e48a2e..538bc0a44 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy @@ -1,9 +1,11 @@ package com.fangjia.script.gj.data_handle +import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import com.rabbitmq.client.impl.AMQImpl import java.util.regex.Pattern @@ -19,15 +21,25 @@ class MinglujiData { DBCollection dev_mapping_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL) DBCollection community_100_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL) + + DBCollection company_100_CONN = MongoUtils.getCollectionByName2("fangjia", "company", MONGO100_URL) + static main(args) { def task = new MinglujiData() task.run() } def run(){ -// exportDatas("湖州") -// matchDev("浙江省","湖州") - matchCommunity("湖州") + def prov = "浙江省" + def city = "嘉兴" +// exportDatas(city) +// matchDev(prov, city) + matchCommunity(city) + +// tempMatchDev(prov) + + +// saveMljData() } def exportDatas(city){ def datas = findMinglujiDatas(city) @@ -35,71 +47,176 @@ class MinglujiData { } def matchDev(prov, city){ def devs = loadAllDevs(prov) - ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\${city}.xlsx",{d-> - if(d['E'] && !d['N'] ){ - def obj = findDevByName(d['E']) - if(!obj){ - obj = findDevByAlias(d['E']) - if(!obj){ - obj = findDebBySimlar(devs, d['E'], d['B']) - if(!obj){ - obj = findDevByPark(d['A'], d['B'], d['E']) - if (!obj) { - }else{ - d['N'] = "园区" + + def i = 1; + while(i<20) { + def fileName = "/home/goojia/gj/mlj_${i}.xlsx" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\数据建设\\名录集\\${city}_${i}.xlsx" + } + File file=new File(fileName); + if(file.exists()){ + ExcelUtils.multiThreadingModify(fileName, { d -> + if (d['D']) { + def street = d['D'] + def splitStreet = "" + if (street) { + if (street.contains(d['C'])) { + def start = street.toString().lastIndexOf(d['C']) + d['C'].size() + def end = street.toString().size() + splitStreet = street.toString().substring(start, end) + } else { + splitStreet = street } + } + d['K'] = splitStreet + } + + if (d['I']) { + def splitCommunity = "" + if (d['C'] && d['I'].toString().contains(d['C'])) { + splitCommunity = d['I'].toString().substring(d['I'].toString().lastIndexOf(d['C']) + d['C'].toString().size(), d['I'].toString().size()) + } else { + splitCommunity = d['I'] + } + d['L'] = splitCommunity + } + + + if (d['E'] && !d['N']) { + def obj = findDevByName(d['E']) + if (!obj) { + obj = findDevByAlias(d['E']) + if (!obj) { + obj = findDebBySimlar(devs, d['E'], d['B']) + if (!obj) { + obj = findDevByPark(d['A'], d['B'], d['E']) + if (!obj) { + } else { + d['N'] = "园区" + } + } else { + d['N'] = "相似" + } + } else { + d['N'] = "别名" + } + } else { + d['N'] = "相同" + } + if (obj) { + d['M'] = obj.name + d['O'] = obj.city + d['P'] = obj.district + } + } + }) + } + i++ + println fileName +" 完成" + } + + } + + def tempMatchDev(prov){ + def devs = loadAllDevs(prov) + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\湖州-开发区-等级.xlsx",{d-> + if(d['U'] && !d['W'] ){ + def obj = findDevByName(d['U']) + if(!obj){ + obj = findDevByAlias(d['U']) + if(!obj){ + obj = findDebBySimlar(devs, d['U'], d['B']) + if(!obj){ + obj = findDevByPark(d['A'], d['B'], d['U']) + if (!obj) { + }else{ + d['X'] = "园区" + } }else{ - d['N'] = "相似" + d['X'] = "相似" } }else{ - d['N'] = "别名" + d['X'] = "别名" } }else{ - d['N'] = "相同" + d['X'] = "相同" } if(obj){ - d['M'] = obj.name - d['O'] = obj.city - d['P'] = obj.district + d['W'] = obj.name + d['Y'] = obj.city + d['Z'] = obj.district } } }) } def matchCommunity(city){ - ExcelUtils.modify("D:\\数据\\数据建设\\名录集\\${city}-没配居委1.xlsx",{d-> - if(d['A'] && d['B'] && d['L'] && !d['Q']){ - def list = findCommunityByName(d['A'], d['B'], d['L']) - if(!list){ - list = findCommunityBySimlar(d['A'], d['B'], d['L']) - if(!list){ + def i = 1; + while(i<20){ + def fileName = "/home/goojia/gj/mlj_${i}.xlsx" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\数据建设\\名录集\\${city}_${i}.xlsx" + } - }else{ - d['T'] = "包含" - } - }else{ - d['T'] = "相同" - } - if(list){ - if(list.size()==1){ - def obj = list[0] - d['Q'] = obj.name - d['R'] = obj.street_name - }else if(list.size()>=2){ - for(def l : list){ - if(d['C']==l.area_name){ - d['Q'] = l.name - d['R'] = l.street_name - break; - } - } - d['S'] = list.size() - } - } - } - }, 0) + File file=new File(fileName); + if(file.exists()){ + ExcelUtils.multiThreadingModify(fileName,{d-> + if(d['A'] && d['B'] && d['L'] && !d['Q']){ + def list = findCommunityByName(d['A'], d['B'], d['L']) + if(!list){ + list = findCommunityBySimlar(d['A'], d['B'], d['L']) + if(!list){ + + }else{ + d['T'] = "包含" + } + }else{ + d['T'] = "相同" + } + if(list){ + if(list.size()==1){ + def obj = list[0] + d['Q'] = obj.name + d['R'] = obj.street_name + d['U'] = obj.area_name + d['V'] = obj.city_name + }else if(list.size()>=2){ + for(def l : list){ + if(d['C']==l.area_name){ + d['Q'] = l.name + d['R'] = l.street_name + d['U'] = l.area_name + d['V'] = l.city_name + break; + } + } + d['S'] = list.size() + } + } + } + }, 0) + println fileName +" 远行完成" + } + i++ + } } + + def saveMljData(){ + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\湖州入库数据.xlsx",{d-> + def obj = new BasicDBObject() + obj.put("province", d['A']) + obj.put("city", d['B']) + obj.put("district", d['C']) + obj.put("name", d['D']) + obj.put("address", d['E']) + obj.put("dev_name", d['F']) + obj.put("committee", d['G']) + obj.put("street", d['H']) + company_100_CONN.save(obj) + }) + } def loadAllDevs(prov){ def q = new BasicDBObject() q.put("province", prov) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy index c2f11e6f8..527a01dea 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy @@ -27,7 +27,7 @@ class QiliangTest { // task.run() def data = [ - 'address': '景县开发区东区', + 'address': '湖州市南太湖高新技术产业园区大港村陈家斗80号', 'type': '1' //1:对公 0:对私 ] def s = task.getProxyResult(data); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy index b8d657c89..0a4f07a1c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy @@ -126,8 +126,7 @@ class GaoDePoiDataProcess extends BaseBatch{ } def sysnchVillage(){ - def buducitys = [ '七台河市', '上海市','万宁市', '三亚市', '三明市', '三门峡市', '上饶市', '东方市', '东莞市', - '东营市', '中卫市', '中山市', '临夏回族自治州', '临汾市', '临沂市', '临沧市', '临高县', '丹东市', + def buducitys = [ '临汾市', '临沂市', '临沧市', '临高县', '丹东市', '丽水市', '丽江市', '乌兰察布市', '乌海市', '乌鲁木齐市', '乐东黎族自治县', '乐山市', '九江市', '云浮市', '五家渠市', '五指山市', '亳州市', '仙桃市', '伊春市', '伊犁哈萨克自治州', '佛山市', '佳木斯市', '保亭黎族苗族自治县', '保定市', '保山市', '信阳市', '儋州市', '克孜勒苏柯尔克孜自治州', @@ -169,7 +168,8 @@ class GaoDePoiDataProcess extends BaseBatch{ '黔东南苗族侗族自治州', '黔南布依族苗族自治州', '黔西南布依族苗族自治州', '齐齐哈尔市', '龙岩市'] //"丽水市","杭州市", - def runbuducitys = ["湖州市","宁波市","温州市","嘉兴市","台州市","绍兴市","金华市","舟山市","衢州市"] + def runbuducitys = ["湖州市","宁波市","温州市","嘉兴市","台州市","绍兴市","金华市","舟山市","衢州市",'七台河市', '上海市','万宁市', '三亚市', '三明市', '三门峡市', '上饶市', '东方市', '东莞市', + '东营市', '中卫市', '中山市', '临夏回族自治州'] for(def city : buducitys){ if(runbuducitys.contains(city)){ continue diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/MongoUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/MongoUtils.groovy index ec6fa0a0b..54d093d7e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/MongoUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/MongoUtils.groovy @@ -71,6 +71,10 @@ class MongoUtils { return com.fangjia.core.util.MongoUtils.getDbCollection(Constants.TEST_FANGJIA_MONGO_URI, db, name) } + if ((domainUrl == "192.168.0.206") && (port == 27020)) { + return com.fangjia.core.util.MongoUtils.getDbCollection(Constants.SEAWATER_MONGO_URI, db, name) + } + def key = db + name + domainUrl + port if (collectionMap.containsKey(key)) { return collectionMap[key]