Merge branch 'script_gj' into 'master'

删除室租金中不存在的小区

See merge request !1031
This commit is contained in:
2024-12-16 15:41:03 +08:00
3 changed files with 378 additions and 2 deletions
@@ -597,9 +597,9 @@ class CommonDatautils {
keys.put("region", 1);
keys.put("name", 1);
keys.put("_id", 1);
DSCursor cursor = getDataStoreConnection().find(q, keys).batchSize(2000);
DSCursor cursor = DbUtils.seaweed.find(q, keys).batchSize(2000);
Iterator<DBObject> iters = cursor.iterator()
def result = [];
def result = new HashSet();
try{
while(iters.hasNext()) {
DBObject obj = iters.next()
@@ -0,0 +1,323 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.json.JsonSlurper
import net.sf.json.JSONObject
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
//补充库中没有详细地址的小区
class AddDistrictAddress extends BaseBatch{
def fullCityMap = CommonDatautils.queryFullNameCity()
def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com")
public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection seaweed_fence100_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_fence", MONGO100_URL)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
runBatch(new AddDistrictAddress(), args)
}
@Override
public void process(Object args) {
// def citys = CommonDatautils.findAllCity()
// def index = 0
// def allrets = []
// for(def c : citys){
// index ++
// def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2", "address"])
// def noRoadInfoDists = noDetailRoadInfoDistricts(districts)
// println c +" "+index +" 原始小区量:"+districts.size() +" 需新增地址量:"+noRoadInfoDists.size()
// noRoadInfoDists = baiduMatch(c, noRoadInfoDists)
// noRoadInfoDists = gaodeMatch(c, noRoadInfoDists)
//
// def rets = []
// rets << ["id", "城市", "区域", "名称", "地址", "百度原地址", "百度标准地址", "高德原地址", "高德标准地址"]
// for(def d : noRoadInfoDists){
// rets << [d._id.toString(), d.city, d.region, d.name, d.address, d.baidu_old_addr?d.baidu_old_addr:'',d.baidu_address?d.baidu_address:'', d.gaode_old_addr?d.gaode_old_addr:'', d.gaode_addr?d.gaode_addr:""]
// }
// allrets.addAll(rets)
// if(allrets.size()>50000){
// ExcelUtils.write("D:\\数据\\小区\\小区模糊地址补充"+index+".xlsx", allrets, "新增地址")
// allrets.clear()
// }
// }
updateDistAddr()
}
def updateDistAddr(){
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\待新增小区地址.xlsx",{d->
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
def udata = new BasicDBObject()
udata.put("address", d['J'])
def addr_alias = dist.addr_alias
if(!dist.addr_alias){
addr_alias = []
}
if(d['K'] && !addr_alias.contains(d['K'])){
addr_alias << d['K']
}
if(addr_alias){
udata.put("addr_alias", addr_alias)
}
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set',udata))
d['L'] = "已更新"
}
})
}
def noDetailRoadInfoDistricts(districts){
def rets = []
for(def d : districts){
if(d.address){
def flag = DataHandleUtils.isStandardRoad(d.address)
if(!flag){
rets << d
}
}else{
rets << d
}
}
return rets;
}
def baiduMatch(city,districts){
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
fullCityName = data.city
}
final CountDownLatch latch = new CountDownLatch(districts.size())
if(fullCityName){
def totalCount = new AtomicLong()
for(def i=0;i<districts.size();i++){
final def d = districts[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def retMap = baiduProcess(d, fullCityName)
if(retMap){
d.baidu_address = retMap.new_addr
d.baidu_old_addr = retMap.old_addr
println "百度补地址 ["+d.city+" "+d.region+" "+d.name+" ] 原地址:"+d.address +" 百度地址:"+retMap.old_addr+" 新地址:"+retMap.new_addr
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
}else{
println city +" 找不到相应的全称"
}
latch.await()
return districts
}
def baiduProcess(d, fullCityName){
def alias = []
alias << d.name
if(d.alias){
alias.addAll(d.alias)
}
def distData = mcaDistMap.get(d.city+"#"+d.region)
def fullDistName = null
if(distData){
fullDistName = distData.district
}
def retMap = [:]
for(def name : alias){
def temp_address = findBaiduDatas(fullCityName, fullDistName, name)
if(temp_address){
retMap.old_addr = temp_address
}
if(temp_address && DataHandleUtils.isStandardRoad(temp_address)){
retMap.new_addr = AddressUtil.formatAddress(fullCityName, temp_address)
continue
}
}
return retMap
}
def findBaiduDatas(city, district, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("district", district)
q.put("name", name)
q.put('std_tag', '房地产;住宅区')
def dbcursor = BAIDU_POI_COLL.find(q)
def address = null
for(def d : dbcursor){
address = d.addr
}
return address;
}
def gaodeMatch(city,districts){
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
fullCityName = data.city
}
final CountDownLatch latch = new CountDownLatch(districts.size())
if(fullCityName){
def totalCount = new AtomicLong()
for(def i=0;i<districts.size();i++){
final def d = districts[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def retMap = gaodeProcess(d, fullCityName)
if(retMap){
d.gaode_addr = retMap.new_addr
d.gaode_old_addr = retMap.old_addr
println "高德补地址 ["+d.city+" "+d.region+" "+d.name+" ] 原地址:"+d.address +" 高德地址:"+retMap.old_addr+" 新地址:"+retMap.new_addr
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
}else{
println city +" 找不到相应的全称"
}
latch.await()
return districts
}
def gaodeProcess(d, fullCityName){
def alias = []
alias << d.name
if(d.alias){
alias.addAll(d.alias)
}
def distData = mcaDistMap.get(d.city+"#"+d.region)
def fullDistName = null
if(distData){
fullDistName = distData.district
}
def retMap = [:]
def address = null
for(def name : alias){
def temp_addr = findGaoDeDatas2(fullCityName, fullDistName, name)
if(temp_addr){
retMap.old_addr = temp_addr
if(temp_addr && DataHandleUtils.isStandardRoad(temp_addr)){
retMap.new_addr = AddressUtil.formatAddress(fullCityName, temp_addr )
continue
}
}
}
return retMap
}
def findGaoDeDatas2(city, region, name){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/didata_map_poi_2024/_search"
def body = """
{
"_source": {
"includes": ["cat", "address", "id","name"]
},
"query": {
"bool": {
"must": [
{
"term": {
"city" : "${city}"
}
},
{
"term": {
"district" : "${region}"
}
},
{
"match": {
"name": {"query": "${name}","operator": "and"}
}
}
// ,{
// "match": {
// "small_type": "住宅"
// }
// }
]
}
}
}
"""
HttpRequest request = new HttpRequest()
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = new JsonSlurper().parseText(res)
def address = null
if(response && response.hits){
def hits = response.hits.hits
if(hits){
def d = hits[0]
if(d._source.cat.contains("住宅")){
def source = d._source
def newName = source.name.replace("-","").replace("·","").replace("(建设中)","")
if(newName == name){
address = source.address
}
}
}
}
return address
}
}
@@ -0,0 +1,53 @@
package com.fangjia.script.gj.district
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
@BatchOwner(Watcher.GaoJuan)
class DistrictStatRentCleanBatch extends BaseBatch{
public static final def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection RENT_DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia", "district_stat_rent", mongoDbUrl)
static main(args) {
runBatch(new DistrictStatRentCleanBatch(), args)
}
@Override
void process(Object args) {
def citys = CommonDatautils.findAllCity()
for(def c : citys){
def keys = CommonDatautils.getSimpleDistricts("district", c)
running(keys, c)
println c +" ÔËÐÐÍê³É"
}
}
def running(keys, c){
def q = new BasicDBObject("city", c).append("district_name", new BasicDBObject('$ne',''))
def fields = new BasicDBObject()
fields.put("city", 1)
fields.put("region", 1)
fields.put("district_name", 1)
def dbcursor = RENT_DIST_STAT_COLL.find(q, fields).batchSize(10000)
def delRets = []
for(def d : dbcursor){
if(d.district_name && !keys.contains(d.city+"@"+d.region+"@"+d.district_name)){
delRets << d._id
}
if(delRets.size()>=100){
def delquery = new BasicDBObject()
delquery.put("_id", new BasicDBObject('$in', delRets))
RENT_DIST_STAT_COLL.remove(delquery)
println new Date()
}
}
}
}