Merge branch 'script_gj' into 'master'
删除室租金中不存在的小区 See merge request !1031
This commit is contained in:
+2
-2
@@ -597,9 +597,9 @@ class CommonDatautils {
|
||||
keys.put("region", 1);
|
||||
keys.put("name", 1);
|
||||
keys.put("_id", 1);
|
||||
DSCursor cursor = getDataStoreConnection().find(q, keys).batchSize(2000);
|
||||
DSCursor cursor = DbUtils.seaweed.find(q, keys).batchSize(2000);
|
||||
Iterator<DBObject> iters = cursor.iterator()
|
||||
def result = [];
|
||||
def result = new HashSet();
|
||||
try{
|
||||
while(iters.hasNext()) {
|
||||
DBObject obj = iters.next()
|
||||
|
||||
+323
@@ -0,0 +1,323 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import groovy.json.JsonSlurper
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
//补充库中没有详细地址的小区
|
||||
class AddDistrictAddress extends BaseBatch{
|
||||
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
|
||||
|
||||
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
|
||||
public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection seaweed_fence100_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_fence", MONGO100_URL)
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new AddDistrictAddress(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
// def citys = CommonDatautils.findAllCity()
|
||||
// def index = 0
|
||||
// def allrets = []
|
||||
// for(def c : citys){
|
||||
// index ++
|
||||
// def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2", "address"])
|
||||
// def noRoadInfoDists = noDetailRoadInfoDistricts(districts)
|
||||
// println c +" "+index +" 原始小区量:"+districts.size() +" 需新增地址量:"+noRoadInfoDists.size()
|
||||
// noRoadInfoDists = baiduMatch(c, noRoadInfoDists)
|
||||
// noRoadInfoDists = gaodeMatch(c, noRoadInfoDists)
|
||||
//
|
||||
// def rets = []
|
||||
// rets << ["id", "城市", "区域", "名称", "地址", "百度原地址", "百度标准地址", "高德原地址", "高德标准地址"]
|
||||
// for(def d : noRoadInfoDists){
|
||||
// rets << [d._id.toString(), d.city, d.region, d.name, d.address, d.baidu_old_addr?d.baidu_old_addr:'',d.baidu_address?d.baidu_address:'', d.gaode_old_addr?d.gaode_old_addr:'', d.gaode_addr?d.gaode_addr:""]
|
||||
// }
|
||||
// allrets.addAll(rets)
|
||||
// if(allrets.size()>50000){
|
||||
// ExcelUtils.write("D:\\数据\\小区\\小区模糊地址补充"+index+".xlsx", allrets, "新增地址")
|
||||
// allrets.clear()
|
||||
// }
|
||||
// }
|
||||
|
||||
updateDistAddr()
|
||||
|
||||
}
|
||||
|
||||
def updateDistAddr(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\待新增小区地址.xlsx",{d->
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
if(dist){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("address", d['J'])
|
||||
def addr_alias = dist.addr_alias
|
||||
if(!dist.addr_alias){
|
||||
addr_alias = []
|
||||
}
|
||||
if(d['K'] && !addr_alias.contains(d['K'])){
|
||||
addr_alias << d['K']
|
||||
}
|
||||
|
||||
if(addr_alias){
|
||||
udata.put("addr_alias", addr_alias)
|
||||
}
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set',udata))
|
||||
d['L'] = "已更新"
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def noDetailRoadInfoDistricts(districts){
|
||||
def rets = []
|
||||
for(def d : districts){
|
||||
if(d.address){
|
||||
def flag = DataHandleUtils.isStandardRoad(d.address)
|
||||
if(!flag){
|
||||
rets << d
|
||||
}
|
||||
}else{
|
||||
rets << d
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def baiduMatch(city,districts){
|
||||
def data = fullCityMap.get(city)
|
||||
def fullCityName = null
|
||||
if(data){
|
||||
fullCityName = data.city
|
||||
}
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
if(fullCityName){
|
||||
def totalCount = new AtomicLong()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def d = districts[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def retMap = baiduProcess(d, fullCityName)
|
||||
if(retMap){
|
||||
d.baidu_address = retMap.new_addr
|
||||
d.baidu_old_addr = retMap.old_addr
|
||||
println "百度补地址 ["+d.city+" "+d.region+" "+d.name+" ] 原地址:"+d.address +" 百度地址:"+retMap.old_addr+" 新地址:"+retMap.new_addr
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}else{
|
||||
println city +" 找不到相应的全称"
|
||||
}
|
||||
latch.await()
|
||||
return districts
|
||||
}
|
||||
|
||||
def baiduProcess(d, fullCityName){
|
||||
def alias = []
|
||||
alias << d.name
|
||||
if(d.alias){
|
||||
alias.addAll(d.alias)
|
||||
}
|
||||
|
||||
def distData = mcaDistMap.get(d.city+"#"+d.region)
|
||||
def fullDistName = null
|
||||
if(distData){
|
||||
fullDistName = distData.district
|
||||
}
|
||||
|
||||
def retMap = [:]
|
||||
|
||||
for(def name : alias){
|
||||
def temp_address = findBaiduDatas(fullCityName, fullDistName, name)
|
||||
if(temp_address){
|
||||
retMap.old_addr = temp_address
|
||||
}
|
||||
if(temp_address && DataHandleUtils.isStandardRoad(temp_address)){
|
||||
retMap.new_addr = AddressUtil.formatAddress(fullCityName, temp_address)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
return retMap
|
||||
}
|
||||
|
||||
|
||||
def findBaiduDatas(city, district, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("district", district)
|
||||
q.put("name", name)
|
||||
q.put('std_tag', '房地产;住宅区')
|
||||
|
||||
def dbcursor = BAIDU_POI_COLL.find(q)
|
||||
def address = null
|
||||
for(def d : dbcursor){
|
||||
address = d.addr
|
||||
}
|
||||
return address;
|
||||
}
|
||||
|
||||
|
||||
def gaodeMatch(city,districts){
|
||||
def data = fullCityMap.get(city)
|
||||
def fullCityName = null
|
||||
if(data){
|
||||
fullCityName = data.city
|
||||
}
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
if(fullCityName){
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def d = districts[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def retMap = gaodeProcess(d, fullCityName)
|
||||
if(retMap){
|
||||
d.gaode_addr = retMap.new_addr
|
||||
d.gaode_old_addr = retMap.old_addr
|
||||
println "高德补地址 ["+d.city+" "+d.region+" "+d.name+" ] 原地址:"+d.address +" 高德地址:"+retMap.old_addr+" 新地址:"+retMap.new_addr
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}else{
|
||||
println city +" 找不到相应的全称"
|
||||
}
|
||||
latch.await()
|
||||
return districts
|
||||
}
|
||||
|
||||
|
||||
def gaodeProcess(d, fullCityName){
|
||||
def alias = []
|
||||
alias << d.name
|
||||
if(d.alias){
|
||||
alias.addAll(d.alias)
|
||||
}
|
||||
|
||||
def distData = mcaDistMap.get(d.city+"#"+d.region)
|
||||
def fullDistName = null
|
||||
if(distData){
|
||||
fullDistName = distData.district
|
||||
}
|
||||
|
||||
def retMap = [:]
|
||||
def address = null
|
||||
for(def name : alias){
|
||||
def temp_addr = findGaoDeDatas2(fullCityName, fullDistName, name)
|
||||
if(temp_addr){
|
||||
retMap.old_addr = temp_addr
|
||||
if(temp_addr && DataHandleUtils.isStandardRoad(temp_addr)){
|
||||
retMap.new_addr = AddressUtil.formatAddress(fullCityName, temp_addr )
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return retMap
|
||||
}
|
||||
|
||||
|
||||
def findGaoDeDatas2(city, region, name){
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.89:9200/didata_map_poi_2024/_search"
|
||||
def body = """
|
||||
{
|
||||
"_source": {
|
||||
"includes": ["cat", "address", "id","name"]
|
||||
},
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"term": {
|
||||
"city" : "${city}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"term": {
|
||||
"district" : "${region}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
"name": {"query": "${name}","operator": "and"}
|
||||
}
|
||||
}
|
||||
// ,{
|
||||
// "match": {
|
||||
// "small_type": "住宅"
|
||||
// }
|
||||
// }
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
def response = new JsonSlurper().parseText(res)
|
||||
|
||||
def address = null
|
||||
if(response && response.hits){
|
||||
def hits = response.hits.hits
|
||||
if(hits){
|
||||
def d = hits[0]
|
||||
if(d._source.cat.contains("住宅")){
|
||||
def source = d._source
|
||||
def newName = source.name.replace("-","").replace("·","").replace("(建设中)","")
|
||||
if(newName == name){
|
||||
address = source.address
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return address
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class DistrictStatRentCleanBatch extends BaseBatch{
|
||||
|
||||
public static final def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection RENT_DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia", "district_stat_rent", mongoDbUrl)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new DistrictStatRentCleanBatch(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
for(def c : citys){
|
||||
def keys = CommonDatautils.getSimpleDistricts("district", c)
|
||||
running(keys, c)
|
||||
println c +" ÔËÐÐÍê³É"
|
||||
}
|
||||
}
|
||||
|
||||
def running(keys, c){
|
||||
def q = new BasicDBObject("city", c).append("district_name", new BasicDBObject('$ne',''))
|
||||
def fields = new BasicDBObject()
|
||||
fields.put("city", 1)
|
||||
fields.put("region", 1)
|
||||
fields.put("district_name", 1)
|
||||
def dbcursor = RENT_DIST_STAT_COLL.find(q, fields).batchSize(10000)
|
||||
def delRets = []
|
||||
for(def d : dbcursor){
|
||||
if(d.district_name && !keys.contains(d.city+"@"+d.region+"@"+d.district_name)){
|
||||
delRets << d._id
|
||||
}
|
||||
if(delRets.size()>=100){
|
||||
def delquery = new BasicDBObject()
|
||||
delquery.put("_id", new BasicDBObject('$in', delRets))
|
||||
RENT_DIST_STAT_COLL.remove(delquery)
|
||||
println new Date()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user