Merge branch 'script_gj' into 'master'

湖州放前面

See merge request !912
This commit is contained in:
2024-08-19 18:49:32 +08:00
2 changed files with 227 additions and 4 deletions
@@ -0,0 +1,221 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.regex.Pattern
/**
* 名录集
*/
class MinglujiData {
DBCollection mlj_COLL = MongoUtils.getCollectionByName("crawl", "kfq_mingluji_com", "192.168.0.206",27020)
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
DBCollection dev_mapping_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
DBCollection community_100_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
static main(args) {
def task = new MinglujiData()
task.run()
}
def run(){
// exportDatas("湖州")
// matchDev("浙江省","湖州")
matchCommunity("湖州")
}
def exportDatas(city){
def datas = findMinglujiDatas(city)
ExcelUtils.write("D:\\数据\\数据建设\\名录集\\${city}.xlsx", datas, "数据")
}
def matchDev(prov, city){
def devs = loadAllDevs(prov)
ExcelUtils.modify("D:\\数据\\数据建设\\名录集\\${city}.xlsx",{d->
if(d['E'] && !d['N'] ){
def obj = findDevByName(d['E'])
if(!obj){
obj = findDevByAlias(d['E'])
if(!obj){
obj = findDebBySimlar(devs, d['E'], d['B'])
if(!obj){
obj = findDevByPark(d['A'], d['B'], d['E'])
if (!obj) {
}else{
d['N'] = "园区"
}
}else{
d['N'] = "相似"
}
}else{
d['N'] = "别名"
}
}else{
d['N'] = "相同"
}
if(obj){
d['M'] = obj.name
d['O'] = obj.city
d['P'] = obj.district
}
}
})
}
def matchCommunity(city){
ExcelUtils.modify("D:\\数据\\数据建设\\名录集\\${city}.xlsx",{d->
if(d['A'] && d['B'] && d['L'] && !d['Q'] && d['L']=="东村委会"){
def list = findCommunityByName(d['A'], d['B'], d['L'])
if(!list){
list = findCommunityBySimlar(d['A'], d['B'], d['L'])
if(!list){
}else{
d['T'] = "包含"
}
}else{
d['T'] = "相同"
}
if(list){
if(list.size()==1){
def obj = list[0]
d['Q'] = obj.name
d['R'] = obj.street_name
}else if(list.size()>=2){
for(def l : list){
if(d['C']==l.area_name){
d['Q'] = l.name
d['R'] = l.street_name
break;
}
}
d['S'] = list.size()
}
}
}
})
}
def loadAllDevs(prov){
def q = new BasicDBObject()
q.put("province", prov)
return development_zone_CONN.find(q).toList()
}
def findDebBySimlar(def devs, def name, def city){
name = name.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园")
def last = null
for(def d : devs){
def alias = null
if(!d.alias){
alias = []
}else{
alias = d.alias
}
alias << d.name
for(def n : alias){
n = n.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园")
if(d.city && d.city.contains(city) && (n.contains(name) || name.contains(n))){
last = d
break
}
}
if(last){
break
}
}
return last
}
def findDevByName(name){
def q = new BasicDBObject()
q.put("name", name)
return development_zone_CONN.findOne(q)
}
def findDevByAlias(name){
def q = new BasicDBObject()
q.put('alias', new BasicDBObject('$in',[name]))
return development_zone_CONN.findOne(q)
}
def findDevBySimlar(name){
def q = new BasicDBObject()
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
return development_zone_CONN.findOne(q)
}
def findDevByPark(prov, city, name){
def q = new BasicDBObject()
q.put('province',new BasicDBObject('$regex', Pattern.compile('^.*'+prov+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
q.put('city',new BasicDBObject('$regex', Pattern.compile('^.*'+city+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
q.put('park',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
q.put("type", "园区")
return dev_mapping_CONN.findOne(q)
}
def findCommunityByName(province, city, name){
def q = new BasicDBObject()
q.put("province_name", province+"省")
q.put("city_name", city+"市")
q.put("name", name)
def dbcursor = community_100_CONN.find(q)
def list = []
for(def d : dbcursor){
list << d
}
return list
}
def findCommunityBySimlar(province, city, name){
def q = new BasicDBObject()
q.put("province_name", province+"省")
q.put("city_name", city+"市")
q.put("name", new BasicDBObject('$regex', Pattern.compile(name+'.*$', Pattern.CASE_INSENSITIVE)))
def dbcursor = community_100_CONN.find(q)
def list = []
for(def d : dbcursor){
list << d
}
return list
}
def findMinglujiDatas(city){
def keys = new BasicDBObject()
keys.put("c_address",1)
keys.put("c_city",1)
keys.put("c_committee",1)
keys.put("c_community",1)
keys.put("c_company",1)
keys.put("c_district",1)
keys.put("c_street",1)
keys.put("kind",1)
keys.put("name",1)
keys.put("province",1)
def datas = mlj_COLL.find(new BasicDBObject("c_city", city), keys)
def rets = []
rets << ["省","城市","区域","街道","开发区名","类型","公司名","社区","居委","地址","拆分街道","拆分居委","匹配开发区名"]
for(def d : datas){
def street = d.c_street
def splitStreet = ""
if(street){
if(street.contains(d.c_district)){
def start = street.toString().lastIndexOf(d.c_district)+d.c_district.size()
def end = street.toString().size()
splitStreet = street.toString().substring(start, end)
}
}
def splitCommunity = ""
if(d.c_committee){
if(d.c_committee.contains(d.c_district)){
splitCommunity = d.c_committee.substring(d.c_committee.lastIndexOf(d.c_district)+d.c_district.size(), d.c_committee.toString().size())
}
}
rets << [d.province, d.c_city, d.c_district, d.c_street, d.name, d.kind, d.c_company, d.c_community, d.c_committee, d.c_address, splitStreet, splitCommunity]
}
return rets
}
}
@@ -168,7 +168,8 @@ class GaoDePoiDataProcess extends BaseBatch{
// '鸡西市', '鹤壁市', '鹤岗市', '鹰潭市', '黄冈市', '黄山市', '黄石市', '黑河市',
// '黔东南苗族侗族自治州', '黔南布依族苗族自治州', '黔西南布依族苗族自治州', '齐齐哈尔市', '龙岩市']
def buducitys = ["丽水市","杭州市","宁波市","温州市","嘉兴市","台州市","湖州市","绍兴市","金华市","舟山市","衢州市"]
//"丽水市","杭州市",
def buducitys = ["湖州市","宁波市","温州市","嘉兴市","台州市","绍兴市","金华市","舟山市","衢州市"]
for(def city : buducitys){
def pois = findAllVillagePoiMap(city)
def rows = createVillageData(city, pois)
@@ -189,7 +190,6 @@ class GaoDePoiDataProcess extends BaseBatch{
// def townMap = findTownRange(city)
def townMap = CommonDatautils.queryTownByCity(city)
def poiKeysFilter = findEsPoiByCity(city)
for(def i=0;i<pois.size();i++){
def p = pois[i]
fixedThreadPool.execute(new Runnable() {
@@ -322,7 +322,6 @@ class GaoDePoiDataProcess extends BaseBatch{
// retMap.small_type = "村庄级地名"
retMap.cat = map.std_tag
def nameArr = map.std_tag.split(";|:")
if(nameArr.size()>=1){
retMap.type = nameArr[0]
@@ -416,6 +415,9 @@ class GaoDePoiDataProcess extends BaseBatch{
}
def findAllVillagePoiMap(city){
def q = new BasicDBObject("city", city);
q.put("std_tag", new BasicDBObject('$exists', true))
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("district", 1)
@@ -424,7 +426,7 @@ class GaoDePoiDataProcess extends BaseBatch{
keys.put("gcj02_x", 1)
keys.put("gcj02_y", 1)
keys.put("std_tag", 1)
def list = BAIDU_POI_COLL.find(new BasicDBObject("city", city), keys).toList()
def list = BAIDU_POI_COLL.find(q, keys).toList()
return list;
}