湖州放前面
This commit is contained in:
+221
@@ -0,0 +1,221 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 名录集
|
||||
*/
|
||||
class MinglujiData {
|
||||
|
||||
DBCollection mlj_COLL = MongoUtils.getCollectionByName("crawl", "kfq_mingluji_com", "192.168.0.206",27020)
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
|
||||
|
||||
DBCollection dev_mapping_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
|
||||
DBCollection community_100_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
|
||||
|
||||
static main(args) {
|
||||
def task = new MinglujiData()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
// exportDatas("湖州")
|
||||
// matchDev("浙江省","湖州")
|
||||
matchCommunity("湖州")
|
||||
}
|
||||
def exportDatas(city){
|
||||
def datas = findMinglujiDatas(city)
|
||||
ExcelUtils.write("D:\\数据\\数据建设\\名录集\\${city}.xlsx", datas, "数据")
|
||||
}
|
||||
def matchDev(prov, city){
|
||||
def devs = loadAllDevs(prov)
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\名录集\\${city}.xlsx",{d->
|
||||
if(d['E'] && !d['N'] ){
|
||||
def obj = findDevByName(d['E'])
|
||||
if(!obj){
|
||||
obj = findDevByAlias(d['E'])
|
||||
if(!obj){
|
||||
obj = findDebBySimlar(devs, d['E'], d['B'])
|
||||
if(!obj){
|
||||
obj = findDevByPark(d['A'], d['B'], d['E'])
|
||||
if (!obj) {
|
||||
}else{
|
||||
d['N'] = "园区"
|
||||
}
|
||||
}else{
|
||||
d['N'] = "相似"
|
||||
}
|
||||
}else{
|
||||
d['N'] = "别名"
|
||||
}
|
||||
}else{
|
||||
d['N'] = "相同"
|
||||
}
|
||||
if(obj){
|
||||
d['M'] = obj.name
|
||||
d['O'] = obj.city
|
||||
d['P'] = obj.district
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def matchCommunity(city){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\名录集\\${city}.xlsx",{d->
|
||||
if(d['A'] && d['B'] && d['L'] && !d['Q'] && d['L']=="东村委会"){
|
||||
def list = findCommunityByName(d['A'], d['B'], d['L'])
|
||||
if(!list){
|
||||
list = findCommunityBySimlar(d['A'], d['B'], d['L'])
|
||||
if(!list){
|
||||
|
||||
}else{
|
||||
d['T'] = "包含"
|
||||
}
|
||||
}else{
|
||||
d['T'] = "相同"
|
||||
}
|
||||
if(list){
|
||||
if(list.size()==1){
|
||||
def obj = list[0]
|
||||
d['Q'] = obj.name
|
||||
d['R'] = obj.street_name
|
||||
}else if(list.size()>=2){
|
||||
for(def l : list){
|
||||
if(d['C']==l.area_name){
|
||||
d['Q'] = l.name
|
||||
d['R'] = l.street_name
|
||||
break;
|
||||
}
|
||||
}
|
||||
d['S'] = list.size()
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def loadAllDevs(prov){
|
||||
def q = new BasicDBObject()
|
||||
q.put("province", prov)
|
||||
return development_zone_CONN.find(q).toList()
|
||||
}
|
||||
|
||||
def findDebBySimlar(def devs, def name, def city){
|
||||
name = name.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园")
|
||||
def last = null
|
||||
for(def d : devs){
|
||||
def alias = null
|
||||
if(!d.alias){
|
||||
alias = []
|
||||
}else{
|
||||
alias = d.alias
|
||||
}
|
||||
alias << d.name
|
||||
for(def n : alias){
|
||||
n = n.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园")
|
||||
if(d.city && d.city.contains(city) && (n.contains(name) || name.contains(n))){
|
||||
last = d
|
||||
break
|
||||
}
|
||||
}
|
||||
if(last){
|
||||
break
|
||||
}
|
||||
}
|
||||
return last
|
||||
}
|
||||
def findDevByName(name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", name)
|
||||
return development_zone_CONN.findOne(q)
|
||||
}
|
||||
|
||||
def findDevByAlias(name){
|
||||
def q = new BasicDBObject()
|
||||
q.put('alias', new BasicDBObject('$in',[name]))
|
||||
return development_zone_CONN.findOne(q)
|
||||
}
|
||||
|
||||
def findDevBySimlar(name){
|
||||
def q = new BasicDBObject()
|
||||
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
return development_zone_CONN.findOne(q)
|
||||
}
|
||||
|
||||
def findDevByPark(prov, city, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put('province',new BasicDBObject('$regex', Pattern.compile('^.*'+prov+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
q.put('city',new BasicDBObject('$regex', Pattern.compile('^.*'+city+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
q.put('park',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
q.put("type", "园区")
|
||||
return dev_mapping_CONN.findOne(q)
|
||||
}
|
||||
|
||||
def findCommunityByName(province, city, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("province_name", province+"省")
|
||||
q.put("city_name", city+"市")
|
||||
q.put("name", name)
|
||||
def dbcursor = community_100_CONN.find(q)
|
||||
def list = []
|
||||
for(def d : dbcursor){
|
||||
list << d
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
def findCommunityBySimlar(province, city, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("province_name", province+"省")
|
||||
q.put("city_name", city+"市")
|
||||
q.put("name", new BasicDBObject('$regex', Pattern.compile(name+'.*$', Pattern.CASE_INSENSITIVE)))
|
||||
def dbcursor = community_100_CONN.find(q)
|
||||
def list = []
|
||||
for(def d : dbcursor){
|
||||
list << d
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
def findMinglujiDatas(city){
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("c_address",1)
|
||||
keys.put("c_city",1)
|
||||
keys.put("c_committee",1)
|
||||
keys.put("c_community",1)
|
||||
keys.put("c_company",1)
|
||||
keys.put("c_district",1)
|
||||
keys.put("c_street",1)
|
||||
keys.put("kind",1)
|
||||
keys.put("name",1)
|
||||
keys.put("province",1)
|
||||
def datas = mlj_COLL.find(new BasicDBObject("c_city", city), keys)
|
||||
def rets = []
|
||||
rets << ["省","城市","区域","街道","开发区名","类型","公司名","社区","居委","地址","拆分街道","拆分居委","匹配开发区名"]
|
||||
for(def d : datas){
|
||||
def street = d.c_street
|
||||
def splitStreet = ""
|
||||
if(street){
|
||||
if(street.contains(d.c_district)){
|
||||
def start = street.toString().lastIndexOf(d.c_district)+d.c_district.size()
|
||||
def end = street.toString().size()
|
||||
splitStreet = street.toString().substring(start, end)
|
||||
}
|
||||
}
|
||||
def splitCommunity = ""
|
||||
if(d.c_committee){
|
||||
if(d.c_committee.contains(d.c_district)){
|
||||
splitCommunity = d.c_committee.substring(d.c_committee.lastIndexOf(d.c_district)+d.c_district.size(), d.c_committee.toString().size())
|
||||
}
|
||||
}
|
||||
rets << [d.province, d.c_city, d.c_district, d.c_street, d.name, d.kind, d.c_company, d.c_community, d.c_committee, d.c_address, splitStreet, splitCommunity]
|
||||
}
|
||||
return rets
|
||||
}
|
||||
}
|
||||
+6
-4
@@ -168,7 +168,8 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
// '鸡西市', '鹤壁市', '鹤岗市', '鹰潭市', '黄冈市', '黄山市', '黄石市', '黑河市',
|
||||
// '黔东南苗族侗族自治州', '黔南布依族苗族自治州', '黔西南布依族苗族自治州', '齐齐哈尔市', '龙岩市']
|
||||
|
||||
def buducitys = ["丽水市","杭州市","宁波市","温州市","嘉兴市","台州市","湖州市","绍兴市","金华市","舟山市","衢州市"]
|
||||
//"丽水市","杭州市",
|
||||
def buducitys = ["湖州市","宁波市","温州市","嘉兴市","台州市","绍兴市","金华市","舟山市","衢州市"]
|
||||
for(def city : buducitys){
|
||||
def pois = findAllVillagePoiMap(city)
|
||||
def rows = createVillageData(city, pois)
|
||||
@@ -189,7 +190,6 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
// def townMap = findTownRange(city)
|
||||
def townMap = CommonDatautils.queryTownByCity(city)
|
||||
def poiKeysFilter = findEsPoiByCity(city)
|
||||
|
||||
for(def i=0;i<pois.size();i++){
|
||||
def p = pois[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
@@ -322,7 +322,6 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
// retMap.small_type = "村庄级地名"
|
||||
retMap.cat = map.std_tag
|
||||
|
||||
|
||||
def nameArr = map.std_tag.split(";|:")
|
||||
if(nameArr.size()>=1){
|
||||
retMap.type = nameArr[0]
|
||||
@@ -416,6 +415,9 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
}
|
||||
|
||||
def findAllVillagePoiMap(city){
|
||||
def q = new BasicDBObject("city", city);
|
||||
q.put("std_tag", new BasicDBObject('$exists', true))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city", 1)
|
||||
keys.put("district", 1)
|
||||
@@ -424,7 +426,7 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
keys.put("gcj02_x", 1)
|
||||
keys.put("gcj02_y", 1)
|
||||
keys.put("std_tag", 1)
|
||||
def list = BAIDU_POI_COLL.find(new BasicDBObject("city", city), keys).toList()
|
||||
def list = BAIDU_POI_COLL.find(q, keys).toList()
|
||||
return list;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user