自动添加小区

This commit is contained in:
2024-10-22 15:22:57 +08:00
parent 8699a0eeda
commit a1a06d0d8f
@@ -0,0 +1,234 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 小区自动入库(新增小区,每月运行一次)
*/
class AutoAddDistrictBatch extends BaseBatch{
final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018)
public static DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def fullCityMap = CommonDatautils.queryFullNameCity()
def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
static main(args) {
runBatch(new AutoAddDistrictBatch(), args)
}
@Override
void process(Object args) {
//查找第三方匹配没有匹配上的小区
def citys = CommonDatautils.findAllCity()
def list = []
list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "高德地图"]
for(def c : citys){
def noMatchDatas = findNoMatchResult(c)
println c +" 新旧算法未匹配的数据量:"+noMatchDatas.size()
def rets = checkDatas(noMatchDatas.keySet())
list.addAll(rets)
}
def fileName = "/home/goojia/gj/add_dist_info.xlsx"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\名录集\\第三方待入库数据.xlsx"
}
ExcelUtils.write(fileName, list, "待入库数据")
}
def checkDatas(keysets){
def noMatchCount = keysets.size()
final CountDownLatch latch = new CountDownLatch(noMatchCount)
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
for(def i=0;i<keysets.size();i++){
final def key = keysets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def arrs = key.split("##")
//检测有没有成交
def trade_count = findTrades(arrs[0], arrs[1], arrs[2])
//检测有没有房源
def house_count = CommonDatautils.findHouseCount(arrs[0], arrs[1], arrs[2])
//检查百度地图有没有
def baidu_count = findBaiduCount(arrs[0], arrs[1], arrs[2])
//检查高德地图有没有
def gaode_count = findGaodeCount(arrs[0], arrs[1], arrs[2])
println "[ "+totalCount.get()+"/"+noMatchCount+" ]"+key +" trade_count:"+trade_count+" house_count:"+house_count+" baidu_count:"+baidu_count+" gaode_count:"+gaode_count
rets << [arrs[0], arrs[1], arrs[2], trade_count, house_count, baidu_count, gaode_count]
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets
}
def findBaiduCount(city, region, name){
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
fullCityName = data.city
}
if(fullCityName){
def distData = mcaDistMap.get(city+"#"+region)
def fullDistName = null
if(distData){
fullDistName = distData.district
}
if(fullDistName){
return findBaiduDatas(fullCityName, fullDistName, name)
}
}
return null
}
def findGaodeCount(city, region, name){
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
fullCityName = data.city
}
if(fullCityName){
def distData = mcaDistMap.get(city+"#"+region)
def fullDistName = null
if(distData){
fullDistName = distData.district
}
if(fullDistName){
def obj = findGaoDeDatas(fullCityName, fullDistName, name)
if(obj){
return 1
}
}
}
return 0
}
def findGaoDeDatas(city, region, name){
String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
def obj = null
ElasticUtils.executeSqlQuery(sql,{ d->
if(d.name == name && d.type.contains("住宅") && d.shape){
obj = d
}
})
return obj;
}
def findTrades(city, region, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("district_name", name)
return DEAL_PRICE_COLL.count(q)
}
def findNoMatchResult(def c){
def q = new BasicDBObject()
q.put("match_id", new BasicDBObject('$exists',false))
q.put("old_match_id", new BasicDBObject('$exists',false))
q.put("city", c)
def dbcursor = thirdSeaweedMappingColl.find(q).batchSize(5000)
def rMap = [:]
for(def d : dbcursor){
if(d.estate_type){
if(d.estate_type.contains("自建房")){
continue
}
if(d.estate_type.contains("购物中心")){
continue
}
if(d.estate_type.contains("住宅底商") || d.estate_type.contains("酒店") || d.estate_type.contains("商铺")){
continue
}
if(d.estate_type.contains("办公") || d.estate_type.contains("写字楼")){
continue
}
}else{
continue
}
def key = d.city + "##"+d.district+"##"+d.name
def rets = rMap.get(key)
if(!rMap.containsKey(key)){
rets = []
rMap.put(key, rets)
}
rets << d
}
return rMap
}
def createData(city, region, name, address, lng, lat){
def udata = new BasicDBObject()
udata.put("city", city)
udata.put("region", region)
name = name.replaceAll('[·\\(\\)()]|·', '').replace("建设中", "")
udata.put("name", name)
def addr = AddressUtil.formatAddress(city, address)
if(addr){
udata.put("address", addr)
}
udata.put("status", "0")
udata.put("name_cat", "0")
udata.put("cat", "district")
udata.put("username", "高娟")
def estateType2 = "普通住宅"
def type = ""
if(type){
if(type.contains("别墅")){
estateType2 = "别墅"
}else if(type.contains("商住")){
estateType2 = "商住"
}
}
udata.put("estateType2", estateType2)
if(lng){
Long gd_lng = lng*10000000000
udata.put("lng", gd_lng)
}
if(lat){
Long gd_lat = lat * 10000000000
udata.put("lat", gd_lat)
}
FangjiaClientUtils.saveOrUpdateDistrict(udata)
}
def findBaiduDatas(city, district, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("district", district)
q.put("name", name)
q.put('std_tag', '房地产;住宅区')
def count = BAIDU_POI_COLL.count(q)
return count;
}
}