diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy new file mode 100644 index 000000000..f27ee361f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy @@ -0,0 +1,234 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.BaseBatch +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 小区自动入库(新增小区,每月运行一次) + */ +class AutoAddDistrictBatch extends BaseBatch{ + + final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018) + public static DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") + DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020) + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + + def fullCityMap = CommonDatautils.queryFullNameCity() + def mcaDistMap = CommonDatautils.queryDistNameCityByMapping() + + static main(args) { + runBatch(new AutoAddDistrictBatch(), args) + } + + @Override + void process(Object args) { + //查找第三方匹配没有匹配上的小区 + def citys = CommonDatautils.findAllCity() + def list = [] + list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "高德地图"] + for(def c : citys){ + def noMatchDatas = findNoMatchResult(c) + println c +" 新旧算法未匹配的数据量:"+noMatchDatas.size() + def rets = checkDatas(noMatchDatas.keySet()) + list.addAll(rets) + } + + def fileName = "/home/goojia/gj/add_dist_info.xlsx" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\数据建设\\名录集\\第三方待入库数据.xlsx" + } + ExcelUtils.write(fileName, list, "待入库数据") + } + + def checkDatas(keysets){ + def noMatchCount = keysets.size() + final CountDownLatch latch = new CountDownLatch(noMatchCount) + + def rets = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + for(def i=0;i + if(d.name == name && d.type.contains("住宅") && d.shape){ + obj = d + } + }) + return obj; + } + + def findTrades(city, region, name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("district_name", name) + return DEAL_PRICE_COLL.count(q) + } + + def findNoMatchResult(def c){ + def q = new BasicDBObject() + q.put("match_id", new BasicDBObject('$exists',false)) + q.put("old_match_id", new BasicDBObject('$exists',false)) + q.put("city", c) + def dbcursor = thirdSeaweedMappingColl.find(q).batchSize(5000) + def rMap = [:] + for(def d : dbcursor){ + if(d.estate_type){ + if(d.estate_type.contains("自建房")){ + continue + } + if(d.estate_type.contains("购物中心")){ + continue + } + if(d.estate_type.contains("住宅底商") || d.estate_type.contains("酒店") || d.estate_type.contains("商铺")){ + continue + } + if(d.estate_type.contains("办公") || d.estate_type.contains("写字楼")){ + continue + } + }else{ + continue + } + def key = d.city + "##"+d.district+"##"+d.name + def rets = rMap.get(key) + if(!rMap.containsKey(key)){ + rets = [] + rMap.put(key, rets) + } + rets << d + } + return rMap + } + + def createData(city, region, name, address, lng, lat){ + def udata = new BasicDBObject() + udata.put("city", city) + udata.put("region", region) + name = name.replaceAll('[·\\(\\)()]|·', '').replace("建设中", "") + udata.put("name", name) + + def addr = AddressUtil.formatAddress(city, address) + if(addr){ + udata.put("address", addr) + } + + udata.put("status", "0") + udata.put("name_cat", "0") + udata.put("cat", "district") + udata.put("username", "高娟") + + def estateType2 = "普通住宅" + def type = "" + if(type){ + if(type.contains("别墅")){ + estateType2 = "别墅" + }else if(type.contains("商住")){ + estateType2 = "商住" + } + } + udata.put("estateType2", estateType2) + if(lng){ + Long gd_lng = lng*10000000000 + udata.put("lng", gd_lng) + } + if(lat){ + Long gd_lat = lat * 10000000000 + udata.put("lat", gd_lat) + } + FangjiaClientUtils.saveOrUpdateDistrict(udata) + } + + def findBaiduDatas(city, district, name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("district", district) + q.put("name", name) + q.put('std_tag', '房地产;住宅区') + def count = BAIDU_POI_COLL.count(q) + return count; + } +}