From a7bb5f2890643f8c0f750531619cab17308dfbd6 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Mon, 19 Aug 2024 18:48:25 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B9=96=E5=B7=9E=E6=94=BE=E5=89=8D=E9=9D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/gj/data_handle/MinglujiData.groovy | 221 ++++++++++++++++++ .../gj/xiamen/GaoDePoiDataProcess.groovy | 10 +- 2 files changed, 227 insertions(+), 4 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy new file mode 100644 index 000000000..e68ef59b1 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy @@ -0,0 +1,221 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.regex.Pattern + +/** + * 名录集 + */ +class MinglujiData { + + DBCollection mlj_COLL = MongoUtils.getCollectionByName("crawl", "kfq_mingluji_com", "192.168.0.206",27020) + def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; + DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL) + + DBCollection dev_mapping_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL) + DBCollection community_100_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL) + + static main(args) { + def task = new MinglujiData() + task.run() + } + + def run(){ +// exportDatas("湖州") +// matchDev("浙江省","湖州") + matchCommunity("湖州") + } + def exportDatas(city){ + def datas = findMinglujiDatas(city) + ExcelUtils.write("D:\\数据\\数据建设\\名录集\\${city}.xlsx", datas, "数据") + } + def matchDev(prov, city){ + def devs = loadAllDevs(prov) + ExcelUtils.modify("D:\\数据\\数据建设\\名录集\\${city}.xlsx",{d-> + if(d['E'] && !d['N'] ){ + def obj = findDevByName(d['E']) + if(!obj){ + obj = findDevByAlias(d['E']) + if(!obj){ + obj = findDebBySimlar(devs, d['E'], d['B']) + if(!obj){ + obj = findDevByPark(d['A'], d['B'], d['E']) + if (!obj) { + }else{ + d['N'] = "园区" + } + }else{ + d['N'] = "相似" + } + }else{ + d['N'] = "别名" + } + }else{ + d['N'] = "相同" + } + if(obj){ + d['M'] = obj.name + d['O'] = obj.city + d['P'] = obj.district + } + } + }) + } + + def matchCommunity(city){ + ExcelUtils.modify("D:\\数据\\数据建设\\名录集\\${city}.xlsx",{d-> + if(d['A'] && d['B'] && d['L'] && !d['Q'] && d['L']=="东村委会"){ + def list = findCommunityByName(d['A'], d['B'], d['L']) + if(!list){ + list = findCommunityBySimlar(d['A'], d['B'], d['L']) + if(!list){ + + }else{ + d['T'] = "包含" + } + }else{ + d['T'] = "相同" + } + if(list){ + if(list.size()==1){ + def obj = list[0] + d['Q'] = obj.name + d['R'] = obj.street_name + }else if(list.size()>=2){ + for(def l : list){ + if(d['C']==l.area_name){ + d['Q'] = l.name + d['R'] = l.street_name + break; + } + } + d['S'] = list.size() + } + } + } + }) + } + + def loadAllDevs(prov){ + def q = new BasicDBObject() + q.put("province", prov) + return development_zone_CONN.find(q).toList() + } + + def findDebBySimlar(def devs, def name, def city){ + name = name.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园") + def last = null + for(def d : devs){ + def alias = null + if(!d.alias){ + alias = [] + }else{ + alias = d.alias + } + alias << d.name + for(def n : alias){ + n = n.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园") + if(d.city && d.city.contains(city) && (n.contains(name) || name.contains(n))){ + last = d + break + } + } + if(last){ + break + } + } + return last + } + def findDevByName(name){ + def q = new BasicDBObject() + q.put("name", name) + return development_zone_CONN.findOne(q) + } + + def findDevByAlias(name){ + def q = new BasicDBObject() + q.put('alias', new BasicDBObject('$in',[name])) + return development_zone_CONN.findOne(q) + } + + def findDevBySimlar(name){ + def q = new BasicDBObject() + q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询 + return development_zone_CONN.findOne(q) + } + + def findDevByPark(prov, city, name){ + def q = new BasicDBObject() + q.put('province',new BasicDBObject('$regex', Pattern.compile('^.*'+prov+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询 + q.put('city',new BasicDBObject('$regex', Pattern.compile('^.*'+city+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询 + q.put('park',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询 + q.put("type", "园区") + return dev_mapping_CONN.findOne(q) + } + + def findCommunityByName(province, city, name){ + def q = new BasicDBObject() + q.put("province_name", province+"省") + q.put("city_name", city+"市") + q.put("name", name) + def dbcursor = community_100_CONN.find(q) + def list = [] + for(def d : dbcursor){ + list << d + } + return list + } + + def findCommunityBySimlar(province, city, name){ + def q = new BasicDBObject() + q.put("province_name", province+"省") + q.put("city_name", city+"市") + q.put("name", new BasicDBObject('$regex', Pattern.compile(name+'.*$', Pattern.CASE_INSENSITIVE))) + def dbcursor = community_100_CONN.find(q) + def list = [] + for(def d : dbcursor){ + list << d + } + return list + } + + def findMinglujiDatas(city){ + def keys = new BasicDBObject() + keys.put("c_address",1) + keys.put("c_city",1) + keys.put("c_committee",1) + keys.put("c_community",1) + keys.put("c_company",1) + keys.put("c_district",1) + keys.put("c_street",1) + keys.put("kind",1) + keys.put("name",1) + keys.put("province",1) + def datas = mlj_COLL.find(new BasicDBObject("c_city", city), keys) + def rets = [] + rets << ["省","城市","区域","街道","开发区名","类型","公司名","社区","居委","地址","拆分街道","拆分居委","匹配开发区名"] + for(def d : datas){ + def street = d.c_street + def splitStreet = "" + if(street){ + if(street.contains(d.c_district)){ + def start = street.toString().lastIndexOf(d.c_district)+d.c_district.size() + def end = street.toString().size() + splitStreet = street.toString().substring(start, end) + } + } + def splitCommunity = "" + if(d.c_committee){ + if(d.c_committee.contains(d.c_district)){ + splitCommunity = d.c_committee.substring(d.c_committee.lastIndexOf(d.c_district)+d.c_district.size(), d.c_committee.toString().size()) + } + } + rets << [d.province, d.c_city, d.c_district, d.c_street, d.name, d.kind, d.c_company, d.c_community, d.c_committee, d.c_address, splitStreet, splitCommunity] + } + return rets + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy index a4ff32dad..06ddf6d39 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy @@ -168,7 +168,8 @@ class GaoDePoiDataProcess extends BaseBatch{ // '鸡西市', '鹤壁市', '鹤岗市', '鹰潭市', '黄冈市', '黄山市', '黄石市', '黑河市', // '黔东南苗族侗族自治州', '黔南布依族苗族自治州', '黔西南布依族苗族自治州', '齐齐哈尔市', '龙岩市'] - def buducitys = ["丽水市","杭州市","宁波市","温州市","嘉兴市","台州市","湖州市","绍兴市","金华市","舟山市","衢州市"] + //"丽水市","杭州市", + def buducitys = ["湖州市","宁波市","温州市","嘉兴市","台州市","绍兴市","金华市","舟山市","衢州市"] for(def city : buducitys){ def pois = findAllVillagePoiMap(city) def rows = createVillageData(city, pois) @@ -189,7 +190,6 @@ class GaoDePoiDataProcess extends BaseBatch{ // def townMap = findTownRange(city) def townMap = CommonDatautils.queryTownByCity(city) def poiKeysFilter = findEsPoiByCity(city) - for(def i=0;i=1){ retMap.type = nameArr[0] @@ -416,6 +415,9 @@ class GaoDePoiDataProcess extends BaseBatch{ } def findAllVillagePoiMap(city){ + def q = new BasicDBObject("city", city); + q.put("std_tag", new BasicDBObject('$exists', true)) + def keys = new BasicDBObject() keys.put("city", 1) keys.put("district", 1) @@ -424,7 +426,7 @@ class GaoDePoiDataProcess extends BaseBatch{ keys.put("gcj02_x", 1) keys.put("gcj02_y", 1) keys.put("std_tag", 1) - def list = BAIDU_POI_COLL.find(new BasicDBObject("city", city), keys).toList() + def list = BAIDU_POI_COLL.find(q, keys).toList() return list; }