自动添加小区
This commit is contained in:
+234
@@ -0,0 +1,234 @@
|
|||||||
|
package com.fangjia.script.gj.data_handle
|
||||||
|
|
||||||
|
import com.fangjia.script.BaseBatch
|
||||||
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
|
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||||
|
import com.fangjia.script.linxsh.util.AddressUtil
|
||||||
|
import com.fangjia.script.util.ElasticUtils
|
||||||
|
import com.fangjia.script.util.ExcelUtils
|
||||||
|
import com.fangjia.script.util.MongoUtils
|
||||||
|
import com.mongodb.BasicDBObject
|
||||||
|
import com.mongodb.DBCollection
|
||||||
|
|
||||||
|
import java.util.concurrent.CopyOnWriteArrayList
|
||||||
|
import java.util.concurrent.CountDownLatch
|
||||||
|
import java.util.concurrent.ExecutorService
|
||||||
|
import java.util.concurrent.Executors
|
||||||
|
import java.util.concurrent.atomic.AtomicLong
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 小区自动入库(新增小区,每月运行一次)
|
||||||
|
*/
|
||||||
|
class AutoAddDistrictBatch extends BaseBatch{
|
||||||
|
|
||||||
|
final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018)
|
||||||
|
public static DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||||
|
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
|
||||||
|
|
||||||
|
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||||
|
|
||||||
|
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||||
|
def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
|
||||||
|
|
||||||
|
static main(args) {
|
||||||
|
runBatch(new AutoAddDistrictBatch(), args)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
void process(Object args) {
|
||||||
|
//查找第三方匹配没有匹配上的小区
|
||||||
|
def citys = CommonDatautils.findAllCity()
|
||||||
|
def list = []
|
||||||
|
list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "高德地图"]
|
||||||
|
for(def c : citys){
|
||||||
|
def noMatchDatas = findNoMatchResult(c)
|
||||||
|
println c +" 新旧算法未匹配的数据量:"+noMatchDatas.size()
|
||||||
|
def rets = checkDatas(noMatchDatas.keySet())
|
||||||
|
list.addAll(rets)
|
||||||
|
}
|
||||||
|
|
||||||
|
def fileName = "/home/goojia/gj/add_dist_info.xlsx"
|
||||||
|
if (CommonDatautils.isWindow()) {
|
||||||
|
fileName = "D:\\数据\\数据建设\\名录集\\第三方待入库数据.xlsx"
|
||||||
|
}
|
||||||
|
ExcelUtils.write(fileName, list, "待入库数据")
|
||||||
|
}
|
||||||
|
|
||||||
|
def checkDatas(keysets){
|
||||||
|
def noMatchCount = keysets.size()
|
||||||
|
final CountDownLatch latch = new CountDownLatch(noMatchCount)
|
||||||
|
|
||||||
|
def rets = new CopyOnWriteArrayList()
|
||||||
|
def totalCount = new AtomicLong()
|
||||||
|
for(def i=0;i<keysets.size();i++){
|
||||||
|
final def key = keysets[i]
|
||||||
|
fixedThreadPool.execute(new Runnable() {
|
||||||
|
void run() {
|
||||||
|
try{
|
||||||
|
totalCount.addAndGet(1)
|
||||||
|
def arrs = key.split("##")
|
||||||
|
//检测有没有成交
|
||||||
|
def trade_count = findTrades(arrs[0], arrs[1], arrs[2])
|
||||||
|
//检测有没有房源
|
||||||
|
def house_count = CommonDatautils.findHouseCount(arrs[0], arrs[1], arrs[2])
|
||||||
|
//检查百度地图有没有
|
||||||
|
def baidu_count = findBaiduCount(arrs[0], arrs[1], arrs[2])
|
||||||
|
//检查高德地图有没有
|
||||||
|
def gaode_count = findGaodeCount(arrs[0], arrs[1], arrs[2])
|
||||||
|
println "[ "+totalCount.get()+"/"+noMatchCount+" ]"+key +" trade_count:"+trade_count+" house_count:"+house_count+" baidu_count:"+baidu_count+" gaode_count:"+gaode_count
|
||||||
|
rets << [arrs[0], arrs[1], arrs[2], trade_count, house_count, baidu_count, gaode_count]
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace()
|
||||||
|
} finally {
|
||||||
|
latch.countDown()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
latch.await()
|
||||||
|
return rets
|
||||||
|
}
|
||||||
|
|
||||||
|
def findBaiduCount(city, region, name){
|
||||||
|
def data = fullCityMap.get(city)
|
||||||
|
def fullCityName = null
|
||||||
|
if(data){
|
||||||
|
fullCityName = data.city
|
||||||
|
}
|
||||||
|
if(fullCityName){
|
||||||
|
def distData = mcaDistMap.get(city+"#"+region)
|
||||||
|
def fullDistName = null
|
||||||
|
if(distData){
|
||||||
|
fullDistName = distData.district
|
||||||
|
}
|
||||||
|
if(fullDistName){
|
||||||
|
return findBaiduDatas(fullCityName, fullDistName, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
def findGaodeCount(city, region, name){
|
||||||
|
def data = fullCityMap.get(city)
|
||||||
|
def fullCityName = null
|
||||||
|
if(data){
|
||||||
|
fullCityName = data.city
|
||||||
|
}
|
||||||
|
if(fullCityName){
|
||||||
|
def distData = mcaDistMap.get(city+"#"+region)
|
||||||
|
def fullDistName = null
|
||||||
|
if(distData){
|
||||||
|
fullDistName = distData.district
|
||||||
|
}
|
||||||
|
if(fullDistName){
|
||||||
|
def obj = findGaoDeDatas(fullCityName, fullDistName, name)
|
||||||
|
if(obj){
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
def findGaoDeDatas(city, region, name){
|
||||||
|
String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
|
||||||
|
def obj = null
|
||||||
|
ElasticUtils.executeSqlQuery(sql,{ d->
|
||||||
|
if(d.name == name && d.type.contains("住宅") && d.shape){
|
||||||
|
obj = d
|
||||||
|
}
|
||||||
|
})
|
||||||
|
return obj;
|
||||||
|
}
|
||||||
|
|
||||||
|
def findTrades(city, region, name){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("city", city)
|
||||||
|
q.put("region", region)
|
||||||
|
q.put("district_name", name)
|
||||||
|
return DEAL_PRICE_COLL.count(q)
|
||||||
|
}
|
||||||
|
|
||||||
|
def findNoMatchResult(def c){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("match_id", new BasicDBObject('$exists',false))
|
||||||
|
q.put("old_match_id", new BasicDBObject('$exists',false))
|
||||||
|
q.put("city", c)
|
||||||
|
def dbcursor = thirdSeaweedMappingColl.find(q).batchSize(5000)
|
||||||
|
def rMap = [:]
|
||||||
|
for(def d : dbcursor){
|
||||||
|
if(d.estate_type){
|
||||||
|
if(d.estate_type.contains("自建房")){
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if(d.estate_type.contains("购物中心")){
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if(d.estate_type.contains("住宅底商") || d.estate_type.contains("酒店") || d.estate_type.contains("商铺")){
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if(d.estate_type.contains("办公") || d.estate_type.contains("写字楼")){
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
def key = d.city + "##"+d.district+"##"+d.name
|
||||||
|
def rets = rMap.get(key)
|
||||||
|
if(!rMap.containsKey(key)){
|
||||||
|
rets = []
|
||||||
|
rMap.put(key, rets)
|
||||||
|
}
|
||||||
|
rets << d
|
||||||
|
}
|
||||||
|
return rMap
|
||||||
|
}
|
||||||
|
|
||||||
|
def createData(city, region, name, address, lng, lat){
|
||||||
|
def udata = new BasicDBObject()
|
||||||
|
udata.put("city", city)
|
||||||
|
udata.put("region", region)
|
||||||
|
name = name.replaceAll('[·\\(\\)()]|·', '').replace("建设中", "")
|
||||||
|
udata.put("name", name)
|
||||||
|
|
||||||
|
def addr = AddressUtil.formatAddress(city, address)
|
||||||
|
if(addr){
|
||||||
|
udata.put("address", addr)
|
||||||
|
}
|
||||||
|
|
||||||
|
udata.put("status", "0")
|
||||||
|
udata.put("name_cat", "0")
|
||||||
|
udata.put("cat", "district")
|
||||||
|
udata.put("username", "高娟")
|
||||||
|
|
||||||
|
def estateType2 = "普通住宅"
|
||||||
|
def type = ""
|
||||||
|
if(type){
|
||||||
|
if(type.contains("别墅")){
|
||||||
|
estateType2 = "别墅"
|
||||||
|
}else if(type.contains("商住")){
|
||||||
|
estateType2 = "商住"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
udata.put("estateType2", estateType2)
|
||||||
|
if(lng){
|
||||||
|
Long gd_lng = lng*10000000000
|
||||||
|
udata.put("lng", gd_lng)
|
||||||
|
}
|
||||||
|
if(lat){
|
||||||
|
Long gd_lat = lat * 10000000000
|
||||||
|
udata.put("lat", gd_lat)
|
||||||
|
}
|
||||||
|
FangjiaClientUtils.saveOrUpdateDistrict(udata)
|
||||||
|
}
|
||||||
|
|
||||||
|
def findBaiduDatas(city, district, name){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("city", city)
|
||||||
|
q.put("district", district)
|
||||||
|
q.put("name", name)
|
||||||
|
q.put('std_tag', '房地产;住宅区')
|
||||||
|
def count = BAIDU_POI_COLL.count(q)
|
||||||
|
return count;
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user