+2
-3
@@ -28,7 +28,7 @@ import java.util.regex.Pattern
|
||||
class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
def curDate = 20210601//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
|
||||
|
||||
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
|
||||
@@ -265,9 +265,8 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def ret = []
|
||||
def data = distCodeMap.get(trade.region)
|
||||
if(!data){
|
||||
return ret
|
||||
}else{
|
||||
println trade.city +" "+trade.region+" 没有找到对应的区域"
|
||||
return ret
|
||||
}
|
||||
ret << cityCodeDatas.get(trade.city)
|
||||
ret << trade._id.toString()
|
||||
|
||||
+1
-2
@@ -12,8 +12,7 @@ import com.mongodb.DBObject
|
||||
|
||||
class CcbCommonUtils {
|
||||
|
||||
|
||||
public static def citys = ["扬州","上海","北京", "重庆","天津","成都","沈阳","大连","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"]
|
||||
public static def citys = [ "扬州","上海","北京","重庆","天津","成都","沈阳","大连","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"]
|
||||
// public static def citys = ["武汉"]
|
||||
|
||||
public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"]
|
||||
|
||||
@@ -259,6 +259,31 @@ class FiledStrand {
|
||||
return updateFile;
|
||||
}
|
||||
|
||||
static def processArea(all_area){
|
||||
if(all_area){
|
||||
def v = getValue(all_area)
|
||||
if(v>=1000 && v<=10000000){
|
||||
|
||||
return v.toString()
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def processCarSite(car_site_count){
|
||||
def v = getValue(car_site_count.toString());
|
||||
if(v){
|
||||
if(v>=1 && v<=100000){
|
||||
try {
|
||||
return v.toString().toInteger();
|
||||
}catch(Exception e){
|
||||
println e.getMessage()
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def addBuildAreaToData(def dist, def updateFile){
|
||||
def build_area = null;
|
||||
if(dist.build_area){
|
||||
@@ -298,6 +323,26 @@ class FiledStrand {
|
||||
}
|
||||
return updateFile;
|
||||
}
|
||||
|
||||
public static def processCubagePercent(cubage_percent, estate_type2){
|
||||
if(cubage_percent){
|
||||
def v = getValue(cubage_percent)
|
||||
if(v){
|
||||
if(estate_type2 == "别墅"){
|
||||
if(v>=0.3 && v<=1){
|
||||
return v.toString()
|
||||
}
|
||||
}else{
|
||||
if(v>1 && v<=10){
|
||||
return v.toString()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
static def addVirescencePercentToData(dist, updateFile){
|
||||
def virescence_percent = null
|
||||
if(dist.virescence_percent){
|
||||
@@ -312,6 +357,16 @@ class FiledStrand {
|
||||
return updateFile;
|
||||
}
|
||||
|
||||
static def processVirescencePercent(virescence_percent){
|
||||
if(virescence_percent){
|
||||
def v = getValue(virescence_percent)
|
||||
if(v && v>15 && v<=70){
|
||||
return v.toString()
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
static def addBuildTypeToData(dist, updateFile){
|
||||
if(dist.estateType2 && dist.buildType){
|
||||
def house = ["超高层","小高层","多层","高层","花园洋房","老洋房","四合院"]
|
||||
|
||||
+268
@@ -0,0 +1,268 @@
|
||||
package com.fangjia.script.gj.data
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.FiledStrand
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import com.rabbitmq.client.AMQP
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class AddDistrictFieldValue {
|
||||
|
||||
def detailColl = CrawalDBUtil.getDistrictDetailColl();
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
|
||||
|
||||
static main(args) {
|
||||
def task = new AddDistrictFieldValue();
|
||||
task.run();
|
||||
}
|
||||
|
||||
def run(){
|
||||
def citys = CcbCommonUtils.citys;
|
||||
// for(def city : citys){
|
||||
// def rets = findAllDataByCity(city)
|
||||
// match(rets)
|
||||
// }
|
||||
|
||||
for(def city : citys){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"])
|
||||
def rMap = getMatchDistrict(city)
|
||||
createResultValue(dists, rMap)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def match(def rets){
|
||||
def totalCount = new AtomicLong()
|
||||
def total = rets.size()
|
||||
final CountDownLatch latch = new CountDownLatch(total)
|
||||
for(def i=0;i<rets.size();i++){
|
||||
final def d = rets[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def region = RegionData.formatRegion(d.city, d.region)
|
||||
if(region) {
|
||||
def rMap = SeaweedDataMatchProcess.doMatch(d.city, region, d.name, d.address);
|
||||
if(rMap){
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", d._id)
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("match_city", rMap.city)
|
||||
udata.put("match_region", rMap.region)
|
||||
udata.put("match_name", rMap.name)
|
||||
detailColl.update(q, new BasicDBObject('$set',udata))
|
||||
}
|
||||
println totalCount.get()+"/${total}"+" "+d.city +" "+d.region+" "+d.name +" MATCH:"+rMap
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
def findAllDataByCity(def city){
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city", 1)
|
||||
keys.put("region", 1)
|
||||
keys.put("name", 1)
|
||||
keys.put("address", 1)
|
||||
keys.put("match_name", 1)
|
||||
|
||||
def dbcursor = detailColl.find(new BasicDBObject("city", city), keys).batchSize(2000)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
if(!d.match_name){
|
||||
rets << d
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def getMatchDistrict(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("match_name", new BasicDBObject('$exists', true))
|
||||
def dbcursor = detailColl.find(q)
|
||||
def rMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = d.match_city +"#"+d.match_region+"#"+d.match_name
|
||||
def sourceMap = rMap.get(key)
|
||||
if(!sourceMap){
|
||||
sourceMap = [:]
|
||||
rMap.put(key, sourceMap)
|
||||
}
|
||||
def value = sourceMap.get(d.root)
|
||||
if(!value){
|
||||
sourceMap.put(d.root, d)
|
||||
}
|
||||
}
|
||||
return rMap;
|
||||
}
|
||||
|
||||
def createResultValue(dists, rMap){
|
||||
def updateIndex = 0
|
||||
def total = dists.size()
|
||||
for(def d : dists){
|
||||
def key = d.city +"#"+d.region+"#"+d.name
|
||||
def sourceMap = rMap.get(key)
|
||||
if(sourceMap){
|
||||
def udata = new BasicDBObject()
|
||||
if(!d.address){
|
||||
def lastValue = getValuesByField(sourceMap,"address", d.estate_type2)
|
||||
if(lastValue){
|
||||
udata.put("address", lastValue)
|
||||
}
|
||||
}
|
||||
if(!d.all_area){
|
||||
def lastValue = getValuesByField(sourceMap,"all_area", d.estate_type2)
|
||||
if(lastValue){
|
||||
udata.put("all_area", lastValue)
|
||||
}
|
||||
}
|
||||
|
||||
if(!d.build_area){
|
||||
def lastValue = getValuesByField(sourceMap,"build_area", d.estate_type2)
|
||||
if(lastValue){
|
||||
udata.put("build_area", lastValue)
|
||||
}
|
||||
}
|
||||
|
||||
if(!d.virescence_percent){
|
||||
def lastValue = getValuesByField(sourceMap,"greening_rate", d.estate_type2)
|
||||
if(lastValue){
|
||||
udata.put("virescence_percent", lastValue)
|
||||
}
|
||||
}
|
||||
|
||||
if(!d.cubage_percent){
|
||||
def lastValue = getValuesByField(sourceMap,"volumetric_rate", d.estate_type2)
|
||||
if(lastValue){
|
||||
udata.put("cubage_percent", lastValue)
|
||||
}
|
||||
}
|
||||
|
||||
if(!d.finish_building_date){
|
||||
def lastValue = getValuesByField(sourceMap,"build_date", d.estate_type2)
|
||||
if(lastValue){
|
||||
udata.put("finish_building_date", lastValue)
|
||||
}
|
||||
}
|
||||
|
||||
if(!d.household_count){
|
||||
def lastValue = getValuesByField(sourceMap,"house_count", d.estate_type2)
|
||||
if(lastValue){
|
||||
udata.put("household_count", lastValue)
|
||||
}
|
||||
}
|
||||
|
||||
if(!d.car_site_count){
|
||||
def lastValue = getValuesByField(sourceMap,"car_site", d.estate_type2)
|
||||
if(lastValue){
|
||||
udata.put("car_site_count", lastValue)
|
||||
}
|
||||
}
|
||||
|
||||
if(udata.size()>0){
|
||||
updateIndex++
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", d._id)
|
||||
println updateIndex +"/${total} "+d.city +" "+d.name+" "+udata
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def getValuesByField(sourceMap, field, estateType){
|
||||
def map = [:]
|
||||
sourceMap.each{k,v->
|
||||
def oldValue = v.get(field);
|
||||
if(oldValue){
|
||||
def newValue = null;
|
||||
if(["address"].contains(field)){
|
||||
newValue = DataUtil.formateAddress(v.get("city"), oldValue)
|
||||
}
|
||||
if(["all_area","build_area"].contains(field)){
|
||||
newValue = FiledStrand.processArea(oldValue)
|
||||
}
|
||||
if(["build_date"].contains(field)){
|
||||
newValue = FiledStrand.processFinishBuildingDate(oldValue)
|
||||
}
|
||||
if(["house_count"].contains(field)){
|
||||
newValue = FiledStrand.processHouseholdCount(oldValue, estateType)
|
||||
}
|
||||
if(["greening_rate"].contains(field)){
|
||||
newValue = FiledStrand.processVirescencePercent(oldValue)
|
||||
}
|
||||
if(["volumetric_rate"].contains(field)){
|
||||
newValue = FiledStrand.processCubagePercent(oldValue, estateType)
|
||||
}
|
||||
if(["car_site_desc","car_site"].contains(field)){
|
||||
newValue = FiledStrand.processCarSite(oldValue)
|
||||
}
|
||||
if(newValue){
|
||||
def count = map.get(newValue)
|
||||
if(!count){
|
||||
map.put(newValue, 1)
|
||||
}else{
|
||||
map.put(newValue, count+1)
|
||||
}
|
||||
|
||||
v.put(field, newValue)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def lastValue = null
|
||||
if(map && map.size()>0){
|
||||
map = map.sort {a, b -> b.value <=> a.value}
|
||||
def values = map.values()
|
||||
if(values.size()==1){
|
||||
lastValue = map.keySet()[0]
|
||||
}else{
|
||||
def count1 = map.values()[0]
|
||||
def count2 = map.values()[1]
|
||||
if(count1 > count2){
|
||||
lastValue = map.keySet()[0]
|
||||
}else{
|
||||
def rets = [map.keySet()[0],map.keySet()[1]]
|
||||
def roots = ["lianjia.com", "ke.com","fang.com","58.com","anjuke.com","5i5j.com","jiwu.com","centanet.com","house365.com","zhuge.com"]
|
||||
for(def root : roots){
|
||||
def data = sourceMap.get(root)
|
||||
if(data){
|
||||
def value = data.get(field)
|
||||
if(rets.contains(value)){
|
||||
lastValue = value
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return lastValue;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user