From f3f790d65b232d0d30a5489ab1c4e9ed61639e7a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 13 Aug 2021 09:54:01 +0800 Subject: [PATCH] update --- .../ccbevaluation/BigdataDealBatch.groovy | 5 +- .../ccbevaluation/CcbCommonUtils.groovy | 3 +- .../com/fangjia/script/gj/FiledStrand.groovy | 55 ++++ .../gj/data/AddDistrictFieldValue.groovy | 268 ++++++++++++++++++ 4 files changed, 326 insertions(+), 5 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/AddDistrictFieldValue.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 5d7d83944..2a413d1b3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -28,7 +28,7 @@ import java.util.regex.Pattern class BigdataDealBatch extends BaseBatch{ def curDate = 20210601//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"] public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; @@ -265,9 +265,8 @@ class BigdataDealBatch extends BaseBatch{ def ret = [] def data = distCodeMap.get(trade.region) if(!data){ - return ret - }else{ println trade.city +" "+trade.region+" 没有找到对应的区域" + return ret } ret << cityCodeDatas.get(trade.city) ret << trade._id.toString() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index f7358c436..5a74c2733 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -12,8 +12,7 @@ import com.mongodb.DBObject class CcbCommonUtils { - - public static def citys = ["扬州","上海","北京", "重庆","天津","成都","沈阳","大连","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"] + public static def citys = [ "扬州","上海","北京","重庆","天津","成都","沈阳","大连","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"] // public static def citys = ["武汉"] public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy index 3fd9abe32..cdbeaefd7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy @@ -259,6 +259,31 @@ class FiledStrand { return updateFile; } + static def processArea(all_area){ + if(all_area){ + def v = getValue(all_area) + if(v>=1000 && v<=10000000){ + + return v.toString() + } + } + return null + } + + static def processCarSite(car_site_count){ + def v = getValue(car_site_count.toString()); + if(v){ + if(v>=1 && v<=100000){ + try { + return v.toString().toInteger(); + }catch(Exception e){ + println e.getMessage() + } + } + } + return null + } + static def addBuildAreaToData(def dist, def updateFile){ def build_area = null; if(dist.build_area){ @@ -298,6 +323,26 @@ class FiledStrand { } return updateFile; } + + public static def processCubagePercent(cubage_percent, estate_type2){ + if(cubage_percent){ + def v = getValue(cubage_percent) + if(v){ + if(estate_type2 == "别墅"){ + if(v>=0.3 && v<=1){ + return v.toString() + } + }else{ + if(v>1 && v<=10){ + return v.toString() + } + } + } + } + return null + } + + static def addVirescencePercentToData(dist, updateFile){ def virescence_percent = null if(dist.virescence_percent){ @@ -312,6 +357,16 @@ class FiledStrand { return updateFile; } + static def processVirescencePercent(virescence_percent){ + if(virescence_percent){ + def v = getValue(virescence_percent) + if(v && v>15 && v<=70){ + return v.toString() + } + } + return null; + } + static def addBuildTypeToData(dist, updateFile){ if(dist.estateType2 && dist.buildType){ def house = ["超高层","小高层","多层","高层","花园洋房","老洋房","四合院"] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/AddDistrictFieldValue.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/AddDistrictFieldValue.groovy new file mode 100644 index 000000000..d91c34642 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/AddDistrictFieldValue.groovy @@ -0,0 +1,268 @@ +package com.fangjia.script.gj.data + +import com.fangjia.core.RegionData +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.FiledStrand +import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil +import com.fangjia.script.util.DataUtil +import com.fangjia.script.util.DbUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import com.rabbitmq.client.AMQP +import com.rabbitmq.client.impl.AMQImpl +import org.bson.types.ObjectId + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class AddDistrictFieldValue { + + def detailColl = CrawalDBUtil.getDistrictDetailColl(); + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + + static main(args) { + def task = new AddDistrictFieldValue(); + task.run(); + } + + def run(){ + def citys = CcbCommonUtils.citys; +// for(def city : citys){ +// def rets = findAllDataByCity(city) +// match(rets) +// } + + for(def city : citys){ + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"]) + def rMap = getMatchDistrict(city) + createResultValue(dists, rMap) + } + + } + + def match(def rets){ + def totalCount = new AtomicLong() + def total = rets.size() + final CountDownLatch latch = new CountDownLatch(total) + for(def i=0;i0){ + updateIndex++ + def q = new BasicDBObject() + q.put("_id", d._id) + println updateIndex +"/${total} "+d.city +" "+d.name+" "+udata + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + + } + } + } + } + + def getValuesByField(sourceMap, field, estateType){ + def map = [:] + sourceMap.each{k,v-> + def oldValue = v.get(field); + if(oldValue){ + def newValue = null; + if(["address"].contains(field)){ + newValue = DataUtil.formateAddress(v.get("city"), oldValue) + } + if(["all_area","build_area"].contains(field)){ + newValue = FiledStrand.processArea(oldValue) + } + if(["build_date"].contains(field)){ + newValue = FiledStrand.processFinishBuildingDate(oldValue) + } + if(["house_count"].contains(field)){ + newValue = FiledStrand.processHouseholdCount(oldValue, estateType) + } + if(["greening_rate"].contains(field)){ + newValue = FiledStrand.processVirescencePercent(oldValue) + } + if(["volumetric_rate"].contains(field)){ + newValue = FiledStrand.processCubagePercent(oldValue, estateType) + } + if(["car_site_desc","car_site"].contains(field)){ + newValue = FiledStrand.processCarSite(oldValue) + } + if(newValue){ + def count = map.get(newValue) + if(!count){ + map.put(newValue, 1) + }else{ + map.put(newValue, count+1) + } + + v.put(field, newValue) + } + } + } + + def lastValue = null + if(map && map.size()>0){ + map = map.sort {a, b -> b.value <=> a.value} + def values = map.values() + if(values.size()==1){ + lastValue = map.keySet()[0] + }else{ + def count1 = map.values()[0] + def count2 = map.values()[1] + if(count1 > count2){ + lastValue = map.keySet()[0] + }else{ + def rets = [map.keySet()[0],map.keySet()[1]] + def roots = ["lianjia.com", "ke.com","fang.com","58.com","anjuke.com","5i5j.com","jiwu.com","centanet.com","house365.com","zhuge.com"] + for(def root : roots){ + def data = sourceMap.get(root) + if(data){ + def value = data.get(field) + if(rets.contains(value)){ + lastValue = value + break + } + } + } + } + } + } + + return lastValue; + } + + +}