Merge branch 'script_gj' of http://gitlab.fangjia.com/dev0/main into script_gj

This commit is contained in:
2024-11-07 16:48:32 +08:00
5 changed files with 181 additions and 71 deletions
@@ -4858,6 +4858,7 @@ class CommonDatautils {
keys.put("city",1) keys.put("city",1)
keys.put("district",1) keys.put("district",1)
keys.put("alias",1) keys.put("alias",1)
keys.put("mzb_district_name",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def retMap = [:] def retMap = [:]
for(def d : dbcursor){ for(def d : dbcursor){
@@ -4866,7 +4867,7 @@ class CommonDatautils {
regionMap = [:] regionMap = [:]
retMap.put(d.city, regionMap) retMap.put(d.city, regionMap)
} }
regionMap.put(d.district, d.alias) regionMap.put(d.district, d)
} }
return retMap return retMap
@@ -4975,18 +4976,22 @@ class CommonDatautils {
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def townMap = [:] def townMap = [:]
for(def d : dbcursor){ for(def d : dbcursor){
def list = townMap.get(d.district) if(d.mzb_district_name){
if(!list){ def list = townMap.get(d.mzb_district_name)
list = [] if(!list){
list = []
}
list << d
townMap.put(d.mzb_district_name, list)
} }
list << d
townMap.put(d.district, list)
} }
println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss") println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss")
return townMap return townMap
} }
static def queryTownByCity(city){ static def queryTownByCity(city){
def q = new BasicDBObject() def q = new BasicDBObject()
@@ -10,7 +10,6 @@ import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import groovy.json.JsonSlurper import groovy.json.JsonSlurper
import net.sf.json.JSONObject import net.sf.json.JSONObject
@@ -19,7 +18,6 @@ import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong import java.util.concurrent.atomic.AtomicLong
/** /**
* 添加高德地图,百度地图围栏 * 添加高德地图,百度地图围栏
*/ */
@@ -32,6 +30,9 @@ class AddDistrictShapeBatch extends BaseBatch{
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020) DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com") public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com")
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection seaweed_fence100_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_fence", MONGO100_URL)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) { static main(args) {
@@ -41,17 +42,20 @@ class AddDistrictShapeBatch extends BaseBatch{
@Override @Override
public void process(Object args) { public void process(Object args) {
// findGaoDeDatas2("苏州市","吴中区", "水岸清华瀚宫" ) // findGaoDeDatas2("苏州市","吴中区", "水岸清华瀚宫" )//测试
def citys = CommonDatautils.findAllCity() // def citys = CommonDatautils.findAllCity()
def index = 1 // def index = 1
for(def c : citys){ // for(def c : citys){
println c +" "+index // println c +" "+index
index ++ // index ++
def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2"]) // def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2"])
// baiduMatch(c, districts) // baiduMatch(c, districts)
gaodeMatch(c, districts) // gaodeMatch(c, districts)
//
// }
} // checkRangeCount()
synch100()
} }
def baiduMatch(city,districts){ def baiduMatch(city,districts){
@@ -373,4 +377,45 @@ class AddDistrictShapeBatch extends BaseBatch{
} }
def checkRangeCount(){
def dbcursor = seaweed_fence_coll.find(new BasicDBObject()).batchSize(100)
def count = 0
def index =0
for(def d : dbcursor){
index ++
if(d.location){
def coordinates = d.location.coordinates
if(coordinates.size()>1){
count++
println count +" "+ d.city +" "+d.name
}
}else{
println d.city + " "+d.name
}
if(index%1000==0){
println index
}
}
}
def synch100(){
def dbcursor = seaweed_fence_coll.find(new BasicDBObject()).batchSize(100)
def rets = []
for(def d : dbcursor) {
rets << d
if(rets.size()%1000==0){
seaweed_fence100_CONN.insert(rets)
rets.clear()
}
}
if(rets.size()>0){
seaweed_fence100_CONN.insert(rets)
rets.clear()
}
}
} }
@@ -37,7 +37,7 @@ class AutoAddDistrictBatch extends BaseBatch{
@Override @Override
void process(Object args) { void process(Object args) {
//查找第三方匹配没有匹配上的小区 //查找第三方匹配没有匹配上的小区
def citys = CommonDatautils.findAllCity() def citys = ["上海"]//CommonDatautils.findAllCity()
def list = [] def list = []
list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "高德地图"] list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "高德地图"]
for(def c : citys){ for(def c : citys){
@@ -23,6 +23,7 @@ class RegionStandardBatch extends BaseBatch{
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity() def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity() def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
static main(args) { static main(args) {
runBatch(new RegionStandardBatch(), args) runBatch(new RegionStandardBatch(), args)
} }
@@ -37,14 +38,15 @@ class RegionStandardBatch extends BaseBatch{
seaweedRegionStandardByCat("district") seaweedRegionStandardByCat("district")
} }
def poiRegionStandard(){ def poiRegionStandard(){
def runCitys = ["重庆","三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德"] def runCitys = ["重庆","三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德"]
def index = 0 def index = 0
def citys = CommonDatautils.findAllCity() def citys = ["攀枝花"]//CommonDatautils.findAllCity()
for(def city : citys){ for(def city : citys){
index ++ index ++
if(index<32){ // if(index<32){
continue // continue
} // }
if(runCitys.contains(city)){ if(runCitys.contains(city)){
continue continue
} }
@@ -74,6 +76,7 @@ class RegionStandardBatch extends BaseBatch{
} }
} }
@@ -195,39 +198,56 @@ class RegionStandardBatch extends BaseBatch{
def poiHaveRegionProcess(city, set, rMap, datas){ def poiHaveRegionProcess(city, set, rMap, datas){
def results = [] def results = []
for(def d : datas){ for(def d : datas){
if(set.contains(d.district)){ def tjjObj = rMap.get(d.district)
if(tjjObj && tjjObj.mzb_district_name && tjjObj.mzb_district_name==d.district){
continue continue
} }
def updateRegion = null def updateRegion = null
for(def key : set){ def updateFlag = true
def alias = rMap.get(key) if(tjjObj){
if(alias.contains(d.district)){ if(tjjObj.mzb_district_name && tjjObj.mzb_district_name!=d.district){
updateRegion = key updateRegion = tjjObj.mzb_district_name
break;
} }
}
if(updateRegion){
def m = [:]
m.put(d.id, ["district":updateRegion])
results << m
if(results.size()%100==0){
updateRegionByEs(results)
results.clear()
}
println d.city +" "+d.district+" "+d.name +" "+d.status+" 区域修改为:"+updateRegion
}else{ }else{
def m = [:] for(def key : set){
m.put(d.id, ["status":1]) def tempObj = rMap.get(key)
results << m def alias = tempObj.alias
if(results.size()%100==0){ if(alias.contains(d.district) || d.district==tempObj.mzb_district_name){
updateRegionByEs(results) if(d.district!=tempObj.mzb_district_name){
results.clear() updateRegion = key
}else{
updateFlag = false
}
break;
}
} }
println d.city +" "+d.district+" "+d.name +" "+d.status+" 状态修改为:删除"
} }
if(updateFlag){
if(updateRegion){
def m = [:]
m.put(d.id, ["district":updateRegion])
results << m
if(results.size()%100==0){
updateRegionByEs(results)
results.clear()
}
println d.city +" "+d.district+" "+d.name +" "+d.status+" 区域修改为:"+updateRegion
}else{
def m = [:]
m.put(d.id, ["status":1])
results << m
if(results.size()%100==0){
updateRegionByEs(results)
results.clear()
}
println d.city +" "+d.district+" "+d.name +" "+d.status+" 状态修改为:删除"
}
}
} }
if(results.size()>0){ if(results.size()>0){
@@ -1,17 +1,17 @@
package com.fangjia.script.gj.data_handle package com.fangjia.script.gj.data_handle
import com.fangjia.script.BaseBatch import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.xiamen.ElasticUtils89 import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import org.apache.commons.io.FileUtils
import org.elasticsearch.action.bulk.BulkRequest import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.update.UpdateRequest import org.elasticsearch.action.update.UpdateRequest
import org.elasticsearch.client.RequestOptions import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType import org.elasticsearch.common.xcontent.XContentType
import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.index.query.QueryBuilders
/** /**
* 区域标准化 * 区域标准化
* 1.小区库乡镇街道标准化,包含小区,商铺,写字楼 * 1.小区库乡镇街道标准化,包含小区,商铺,写字楼
@@ -23,12 +23,7 @@ class TownStandardBatch extends BaseBatch{
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity() def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity() def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
def poiEsUrl = "http://192.168.0.89:9200/didata_map_poi/_search" def updateList = []
def elasticsearchUrl = "http://192.168.0.89:9200/_search/scroll"
def initialSearchUrl = "http://192.168.0.89:9200/didata_map_poi/_search?scroll=1m"
def scrollTime = "1m" // Scroll context time-to-live
def pageSize = 1000 // Number of hits to return per scroll
static main(args) { static main(args) {
runBatch(new TownStandardBatch(), args) runBatch(new TownStandardBatch(), args)
@@ -70,7 +65,8 @@ class TownStandardBatch extends BaseBatch{
} }
def townStandardByPoi(){ def townStandardByPoi(){
def citys = ["重庆","龙岩","南平","漳州","福州","厦门","泉州","莆田","三明","宁德"]//CommonDatautils.findAllCity() updateList << ["数据ID", "城市","区域","乡镇街道","名称","状态", "修改"]
def citys = CommonDatautils.findAllCity()
for(def city : citys){ for(def city : citys){
def data = fullCityMap.get(city) def data = fullCityMap.get(city)
@@ -79,22 +75,33 @@ class TownStandardBatch extends BaseBatch{
fullCityName = data.city fullCityName = data.city
} }
if(fullCityName){ if(fullCityName){
def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district", "street", "alias"])
def rMap = regionFullNameMap.get(fullCityName) def fileName = "/home/goojia/gj/poi/${fullCityName}.csv"
if(rMap){ if (CommonDatautils.isWindow()) {
def set = rMap.keySet() fileName = "D:\\数据\\${fullCityName}.csv"
def rets = findEsPoiByCity(fullCityName) }
println city +" "+rets.size() File file=new File(fileName);
if(set.size()>1){ if(file.exists()){
poiHaveRegionProcess(fullCityName, townMap, rets) def rMap = regionFullNameMap.get(fullCityName)
}else{ if(rMap){
def mzbRegionMap = CommonDatautils.queryMzbRegion(fullCityName)
def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district","mzb_district_name", "street", "alias"])
def set = rMap.keySet()
// def rets = findEsPoiByCity(fullCityName)
def rets = readPoiCsv(fullCityName)
println city +" "+rets.size()
if(set.size()>1){
poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap)
}else{
// haveStreetProcess(city, set, cat) // haveStreetProcess(city, set, cat)
}
} }
} }
}else{ }else{
println "没找到相应的全称:"+city println "没找到相应的全称:"+city
} }
} }
ExcelUtils.write("D:\\数据\\乡镇街道待查数据.xlsx", updateList, "数据")
} }
def haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap){ def haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap){
def results = [] def results = []
@@ -149,12 +156,15 @@ class TownStandardBatch extends BaseBatch{
} }
} }
} }
} }
def poiHaveRegionProcess(fullCityName, townMap, rets){ def poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap){
def results = [] def results = []
if(fullCityName){ if(fullCityName){
for(def d : rets){ for(def d : rets){
if(d.township){ if(d.township){
def towns = null def towns = null
def updateTown = null def updateTown = null
@@ -163,6 +173,8 @@ class TownStandardBatch extends BaseBatch{
if(townDatas){ if(townDatas){
towns = townDatas towns = townDatas
break break
}else{
println(d.city +" "+d.district+" 没找下辖街道")
} }
} }
@@ -193,6 +205,7 @@ class TownStandardBatch extends BaseBatch{
// updateByEs(results) // updateByEs(results)
// results.clear() // results.clear()
// } // }
updateList << [d.id, d.city, d.district, d.name,d.township, d.status, updateTown]
println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown
}else{ }else{
def m = [:] def m = [:]
@@ -202,6 +215,7 @@ class TownStandardBatch extends BaseBatch{
// updateByEs(results) // updateByEs(results)
// results.clear() // results.clear()
// } // }
updateList << [d.id, d.city, d.district, d.name, d.township, d.status, ""]
println d.city +" "+d.district+" "+d.name +" "+d.township+" 状态修改为:空值" println d.city +" "+d.district+" "+d.name +" "+d.township+" 状态修改为:空值"
} }
} }
@@ -210,10 +224,10 @@ class TownStandardBatch extends BaseBatch{
} }
} }
if(results.size()>0){ // if(results.size()>0){
updateByEs(results) // updateByEs(results)
results.clear() // results.clear()
} // }
println fullCityName +" 处理完成" println fullCityName +" 处理完成"
} }
@@ -276,4 +290,30 @@ class TownStandardBatch extends BaseBatch{
} }
def readPoiCsv(filename){
def datas = readCSV(filename)
return datas
}
def readCSV(filename){
def lines = FileUtils.readLines(new File(filename))
def rets = [];
for(line in lines) {
if(line){
String[] arrs = line.split("\\|@\\|")
def o = new BasicDBObject()
o.put("city", arrs[0])
o.put("district", arrs[1])
o.put("name", arrs[2])
o.put("township", arrs[3])
o.put("status", arrs[4])
o.put("id", arrs[5])
rets << o
}
}
return rets;
}
} }