自动添加小区
This commit is contained in:
+234
@@ -0,0 +1,234 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 小区自动入库(新增小区,每月运行一次)
|
||||
*/
|
||||
class AutoAddDistrictBatch extends BaseBatch{
|
||||
|
||||
final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018)
|
||||
public static DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
|
||||
|
||||
static main(args) {
|
||||
runBatch(new AutoAddDistrictBatch(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
//查找第三方匹配没有匹配上的小区
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def list = []
|
||||
list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "高德地图"]
|
||||
for(def c : citys){
|
||||
def noMatchDatas = findNoMatchResult(c)
|
||||
println c +" 新旧算法未匹配的数据量:"+noMatchDatas.size()
|
||||
def rets = checkDatas(noMatchDatas.keySet())
|
||||
list.addAll(rets)
|
||||
}
|
||||
|
||||
def fileName = "/home/goojia/gj/add_dist_info.xlsx"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\数据建设\\名录集\\第三方待入库数据.xlsx"
|
||||
}
|
||||
ExcelUtils.write(fileName, list, "待入库数据")
|
||||
}
|
||||
|
||||
def checkDatas(keysets){
|
||||
def noMatchCount = keysets.size()
|
||||
final CountDownLatch latch = new CountDownLatch(noMatchCount)
|
||||
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
for(def i=0;i<keysets.size();i++){
|
||||
final def key = keysets[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def arrs = key.split("##")
|
||||
//检测有没有成交
|
||||
def trade_count = findTrades(arrs[0], arrs[1], arrs[2])
|
||||
//检测有没有房源
|
||||
def house_count = CommonDatautils.findHouseCount(arrs[0], arrs[1], arrs[2])
|
||||
//检查百度地图有没有
|
||||
def baidu_count = findBaiduCount(arrs[0], arrs[1], arrs[2])
|
||||
//检查高德地图有没有
|
||||
def gaode_count = findGaodeCount(arrs[0], arrs[1], arrs[2])
|
||||
println "[ "+totalCount.get()+"/"+noMatchCount+" ]"+key +" trade_count:"+trade_count+" house_count:"+house_count+" baidu_count:"+baidu_count+" gaode_count:"+gaode_count
|
||||
rets << [arrs[0], arrs[1], arrs[2], trade_count, house_count, baidu_count, gaode_count]
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
def findBaiduCount(city, region, name){
|
||||
def data = fullCityMap.get(city)
|
||||
def fullCityName = null
|
||||
if(data){
|
||||
fullCityName = data.city
|
||||
}
|
||||
if(fullCityName){
|
||||
def distData = mcaDistMap.get(city+"#"+region)
|
||||
def fullDistName = null
|
||||
if(distData){
|
||||
fullDistName = distData.district
|
||||
}
|
||||
if(fullDistName){
|
||||
return findBaiduDatas(fullCityName, fullDistName, name)
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def findGaodeCount(city, region, name){
|
||||
def data = fullCityMap.get(city)
|
||||
def fullCityName = null
|
||||
if(data){
|
||||
fullCityName = data.city
|
||||
}
|
||||
if(fullCityName){
|
||||
def distData = mcaDistMap.get(city+"#"+region)
|
||||
def fullDistName = null
|
||||
if(distData){
|
||||
fullDistName = distData.district
|
||||
}
|
||||
if(fullDistName){
|
||||
def obj = findGaoDeDatas(fullCityName, fullDistName, name)
|
||||
if(obj){
|
||||
return 1
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
def findGaoDeDatas(city, region, name){
|
||||
String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
|
||||
def obj = null
|
||||
ElasticUtils.executeSqlQuery(sql,{ d->
|
||||
if(d.name == name && d.type.contains("住宅") && d.shape){
|
||||
obj = d
|
||||
}
|
||||
})
|
||||
return obj;
|
||||
}
|
||||
|
||||
def findTrades(city, region, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("district_name", name)
|
||||
return DEAL_PRICE_COLL.count(q)
|
||||
}
|
||||
|
||||
def findNoMatchResult(def c){
|
||||
def q = new BasicDBObject()
|
||||
q.put("match_id", new BasicDBObject('$exists',false))
|
||||
q.put("old_match_id", new BasicDBObject('$exists',false))
|
||||
q.put("city", c)
|
||||
def dbcursor = thirdSeaweedMappingColl.find(q).batchSize(5000)
|
||||
def rMap = [:]
|
||||
for(def d : dbcursor){
|
||||
if(d.estate_type){
|
||||
if(d.estate_type.contains("自建房")){
|
||||
continue
|
||||
}
|
||||
if(d.estate_type.contains("购物中心")){
|
||||
continue
|
||||
}
|
||||
if(d.estate_type.contains("住宅底商") || d.estate_type.contains("酒店") || d.estate_type.contains("商铺")){
|
||||
continue
|
||||
}
|
||||
if(d.estate_type.contains("办公") || d.estate_type.contains("写字楼")){
|
||||
continue
|
||||
}
|
||||
}else{
|
||||
continue
|
||||
}
|
||||
def key = d.city + "##"+d.district+"##"+d.name
|
||||
def rets = rMap.get(key)
|
||||
if(!rMap.containsKey(key)){
|
||||
rets = []
|
||||
rMap.put(key, rets)
|
||||
}
|
||||
rets << d
|
||||
}
|
||||
return rMap
|
||||
}
|
||||
|
||||
def createData(city, region, name, address, lng, lat){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("city", city)
|
||||
udata.put("region", region)
|
||||
name = name.replaceAll('[·\\(\\)()]|·', '').replace("建设中", "")
|
||||
udata.put("name", name)
|
||||
|
||||
def addr = AddressUtil.formatAddress(city, address)
|
||||
if(addr){
|
||||
udata.put("address", addr)
|
||||
}
|
||||
|
||||
udata.put("status", "0")
|
||||
udata.put("name_cat", "0")
|
||||
udata.put("cat", "district")
|
||||
udata.put("username", "高娟")
|
||||
|
||||
def estateType2 = "普通住宅"
|
||||
def type = ""
|
||||
if(type){
|
||||
if(type.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(type.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}
|
||||
}
|
||||
udata.put("estateType2", estateType2)
|
||||
if(lng){
|
||||
Long gd_lng = lng*10000000000
|
||||
udata.put("lng", gd_lng)
|
||||
}
|
||||
if(lat){
|
||||
Long gd_lat = lat * 10000000000
|
||||
udata.put("lat", gd_lat)
|
||||
}
|
||||
FangjiaClientUtils.saveOrUpdateDistrict(udata)
|
||||
}
|
||||
|
||||
def findBaiduDatas(city, district, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("district", district)
|
||||
q.put("name", name)
|
||||
q.put('std_tag', '房地产;住宅区')
|
||||
def count = BAIDU_POI_COLL.count(q)
|
||||
return count;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user