This commit is contained in:
2021-08-13 09:54:01 +08:00
parent e4e08ac004
commit f3f790d65b
4 changed files with 326 additions and 5 deletions
@@ -28,7 +28,7 @@ import java.util.regex.Pattern
class BigdataDealBatch extends BaseBatch{
def curDate = 20210601//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
@@ -265,9 +265,8 @@ class BigdataDealBatch extends BaseBatch{
def ret = []
def data = distCodeMap.get(trade.region)
if(!data){
return ret
}else{
println trade.city +" "+trade.region+" 没有找到对应的区域"
return ret
}
ret << cityCodeDatas.get(trade.city)
ret << trade._id.toString()
@@ -12,8 +12,7 @@ import com.mongodb.DBObject
class CcbCommonUtils {
public static def citys = ["扬州","上海","北京", "重庆","天津","成都","沈阳","大连","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"]
public static def citys = [ "扬州","上海","北京","重庆","天津","成都","沈阳","大连","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"]
// public static def citys = ["武汉"]
public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"]
@@ -259,6 +259,31 @@ class FiledStrand {
return updateFile;
}
static def processArea(all_area){
if(all_area){
def v = getValue(all_area)
if(v>=1000 && v<=10000000){
return v.toString()
}
}
return null
}
static def processCarSite(car_site_count){
def v = getValue(car_site_count.toString());
if(v){
if(v>=1 && v<=100000){
try {
return v.toString().toInteger();
}catch(Exception e){
println e.getMessage()
}
}
}
return null
}
static def addBuildAreaToData(def dist, def updateFile){
def build_area = null;
if(dist.build_area){
@@ -298,6 +323,26 @@ class FiledStrand {
}
return updateFile;
}
public static def processCubagePercent(cubage_percent, estate_type2){
if(cubage_percent){
def v = getValue(cubage_percent)
if(v){
if(estate_type2 == "别墅"){
if(v>=0.3 && v<=1){
return v.toString()
}
}else{
if(v>1 && v<=10){
return v.toString()
}
}
}
}
return null
}
static def addVirescencePercentToData(dist, updateFile){
def virescence_percent = null
if(dist.virescence_percent){
@@ -312,6 +357,16 @@ class FiledStrand {
return updateFile;
}
static def processVirescencePercent(virescence_percent){
if(virescence_percent){
def v = getValue(virescence_percent)
if(v && v>15 && v<=70){
return v.toString()
}
}
return null;
}
static def addBuildTypeToData(dist, updateFile){
if(dist.estateType2 && dist.buildType){
def house = ["超高层","小高层","多层","高层","花园洋房","老洋房","四合院"]
@@ -0,0 +1,268 @@
package com.fangjia.script.gj.data
import com.fangjia.core.RegionData
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.FiledStrand
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import com.rabbitmq.client.AMQP
import com.rabbitmq.client.impl.AMQImpl
import org.bson.types.ObjectId
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class AddDistrictFieldValue {
def detailColl = CrawalDBUtil.getDistrictDetailColl();
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
static main(args) {
def task = new AddDistrictFieldValue();
task.run();
}
def run(){
def citys = CcbCommonUtils.citys;
// for(def city : citys){
// def rets = findAllDataByCity(city)
// match(rets)
// }
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"])
def rMap = getMatchDistrict(city)
createResultValue(dists, rMap)
}
}
def match(def rets){
def totalCount = new AtomicLong()
def total = rets.size()
final CountDownLatch latch = new CountDownLatch(total)
for(def i=0;i<rets.size();i++){
final def d = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def region = RegionData.formatRegion(d.city, d.region)
if(region) {
def rMap = SeaweedDataMatchProcess.doMatch(d.city, region, d.name, d.address);
if(rMap){
def q = new BasicDBObject()
q.put("_id", d._id)
def udata = new BasicDBObject()
udata.put("match_city", rMap.city)
udata.put("match_region", rMap.region)
udata.put("match_name", rMap.name)
detailColl.update(q, new BasicDBObject('$set',udata))
}
println totalCount.get()+"/${total}"+" "+d.city +" "+d.region+" "+d.name +" MATCH:"+rMap
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
def findAllDataByCity(def city){
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("region", 1)
keys.put("name", 1)
keys.put("address", 1)
keys.put("match_name", 1)
def dbcursor = detailColl.find(new BasicDBObject("city", city), keys).batchSize(2000)
def rets = []
for(def d : dbcursor){
if(!d.match_name){
rets << d
}
}
return rets;
}
def getMatchDistrict(def city){
def q = new BasicDBObject()
q.put("city", city)
q.put("match_name", new BasicDBObject('$exists', true))
def dbcursor = detailColl.find(q)
def rMap = [:]
for(def d : dbcursor){
def key = d.match_city +"#"+d.match_region+"#"+d.match_name
def sourceMap = rMap.get(key)
if(!sourceMap){
sourceMap = [:]
rMap.put(key, sourceMap)
}
def value = sourceMap.get(d.root)
if(!value){
sourceMap.put(d.root, d)
}
}
return rMap;
}
def createResultValue(dists, rMap){
def updateIndex = 0
def total = dists.size()
for(def d : dists){
def key = d.city +"#"+d.region+"#"+d.name
def sourceMap = rMap.get(key)
if(sourceMap){
def udata = new BasicDBObject()
if(!d.address){
def lastValue = getValuesByField(sourceMap,"address", d.estate_type2)
if(lastValue){
udata.put("address", lastValue)
}
}
if(!d.all_area){
def lastValue = getValuesByField(sourceMap,"all_area", d.estate_type2)
if(lastValue){
udata.put("all_area", lastValue)
}
}
if(!d.build_area){
def lastValue = getValuesByField(sourceMap,"build_area", d.estate_type2)
if(lastValue){
udata.put("build_area", lastValue)
}
}
if(!d.virescence_percent){
def lastValue = getValuesByField(sourceMap,"greening_rate", d.estate_type2)
if(lastValue){
udata.put("virescence_percent", lastValue)
}
}
if(!d.cubage_percent){
def lastValue = getValuesByField(sourceMap,"volumetric_rate", d.estate_type2)
if(lastValue){
udata.put("cubage_percent", lastValue)
}
}
if(!d.finish_building_date){
def lastValue = getValuesByField(sourceMap,"build_date", d.estate_type2)
if(lastValue){
udata.put("finish_building_date", lastValue)
}
}
if(!d.household_count){
def lastValue = getValuesByField(sourceMap,"house_count", d.estate_type2)
if(lastValue){
udata.put("household_count", lastValue)
}
}
if(!d.car_site_count){
def lastValue = getValuesByField(sourceMap,"car_site", d.estate_type2)
if(lastValue){
udata.put("car_site_count", lastValue)
}
}
if(udata.size()>0){
updateIndex++
def q = new BasicDBObject()
q.put("_id", d._id)
println updateIndex +"/${total} "+d.city +" "+d.name+" "+udata
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}
}
}
}
def getValuesByField(sourceMap, field, estateType){
def map = [:]
sourceMap.each{k,v->
def oldValue = v.get(field);
if(oldValue){
def newValue = null;
if(["address"].contains(field)){
newValue = DataUtil.formateAddress(v.get("city"), oldValue)
}
if(["all_area","build_area"].contains(field)){
newValue = FiledStrand.processArea(oldValue)
}
if(["build_date"].contains(field)){
newValue = FiledStrand.processFinishBuildingDate(oldValue)
}
if(["house_count"].contains(field)){
newValue = FiledStrand.processHouseholdCount(oldValue, estateType)
}
if(["greening_rate"].contains(field)){
newValue = FiledStrand.processVirescencePercent(oldValue)
}
if(["volumetric_rate"].contains(field)){
newValue = FiledStrand.processCubagePercent(oldValue, estateType)
}
if(["car_site_desc","car_site"].contains(field)){
newValue = FiledStrand.processCarSite(oldValue)
}
if(newValue){
def count = map.get(newValue)
if(!count){
map.put(newValue, 1)
}else{
map.put(newValue, count+1)
}
v.put(field, newValue)
}
}
}
def lastValue = null
if(map && map.size()>0){
map = map.sort {a, b -> b.value <=> a.value}
def values = map.values()
if(values.size()==1){
lastValue = map.keySet()[0]
}else{
def count1 = map.values()[0]
def count2 = map.values()[1]
if(count1 > count2){
lastValue = map.keySet()[0]
}else{
def rets = [map.keySet()[0],map.keySet()[1]]
def roots = ["lianjia.com", "ke.com","fang.com","58.com","anjuke.com","5i5j.com","jiwu.com","centanet.com","house365.com","zhuge.com"]
for(def root : roots){
def data = sourceMap.get(root)
if(data){
def value = data.get(field)
if(rets.contains(value)){
lastValue = value
break
}
}
}
}
}
}
return lastValue;
}
}