diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 425f25225..c06cef98e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -35,7 +35,7 @@ class BigdataCommunityBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = ["临夏"]//CcbCommonUtils.citys_all; + def citys = CcbCommonUtils.citys_92; def rows = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 95ec61b0d..6ed03664a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -74,8 +74,8 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); -// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220818, 20220919); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220918, 20221019); +// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); //测试通过匹配接口配房源效果 // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index e5281ae53..3be6ffa9a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -473,6 +473,7 @@ class CommonDatautils { if(dbo.mapping_alias){ detailMap.mapping_alias = dbo.mapping_alias; } + detailMap.estate_type = dbo.estate_type2 } }catch(Exception e){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index 9a9fd90ad..721bf7025 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.thirdPriceCrawl +import com.fangjia.core.util.DateUtil import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher @@ -11,7 +12,7 @@ import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.CosineSimilarAlgorithm import com.fangjia.script.util.DataUtil import com.fangjia.script.util.DbUtils - +import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject @@ -77,6 +78,9 @@ class DataStandardBatch extends BaseBatch{ } def rowIndex = 0; + + DBCollection THIRD_MATCH_FILTER_COLL = MongoUtils.getCollectionByName("fangjia_craw", "third_match_filter", "192.168.0.235", 27018) + DBCollection THIRD_PRICE_FILTER_COLL = MongoUtils.getCollectionByName("fangjia_craw", "third_price_filter", "192.168.0.235", 27018) static main(args) { runBatch(new DataStandardBatch(), args) } @@ -87,9 +91,11 @@ class DataStandardBatch extends BaseBatch{ // ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl) cityMap.each{city,alias-> +// if(city == "北京"){ println "开始加载【${city}】" - def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias"]) + def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"]) districtMap.putAll(detailMap) +// } } while(flag){ @@ -370,39 +376,44 @@ class DataStandardBatch extends BaseBatch{ if(data.name.contains("写字楼")||data.name.contains("办公楼")||data.name.contains("商业")){ district = null; }else{ - district = getMatchNameFromMap(data.city, data.region, data.name, data.address) + district = getMatchNameFromMap(data) if(!district && data.name.length()==2){//没有匹配上,小区名只有2个字就把小区名加“小区” - district = getMatchNameFromMap(data.city, data.region, data.name+"小区", data.address) + def copyObj = data + copyObj.name = data.name+"小区" + district = getMatchNameFromMap(copyObj) } - if(!district){ def newName = filterSpacialChar(data.name);//包含特殊符号 if(newName) { - district = getMatchNameFromMap(data.city, data.region, newName.replaceAll("_", ""), data.address) + def copyObj = data + copyObj.name = newName.replaceAll("_", "") + district = getMatchNameFromMap(copyObj) if(!district){ def s1 = newName.substring(0, newName.indexOf("_")) if (s1) { //特殊符号前的内容匹配 //district = getMatchName(data.city, data.region, s1, data.address,"district") - district = getMatchNameFromMap(data.city, data.region, s1, data.address) + copyObj.name = s1 + district = getMatchNameFromMap(copyObj) if (!district) { - def s2 = newName.replace(s1, "").replaceAll("_", "") def newName2 = null; if (s2 && s2.length() >= 3) { //特殊符号后的内容匹配 // district = getMatchName(data.city, data.region, s2, data.address,"district") - district = getMatchNameFromMap(data.city, data.region, s2, data.address) + copyObj.name = s2 + district = getMatchNameFromMap(copyObj) if (district) { type = 3; println "[匹配成功](后)[${district.city}-${district.region}-${district.name}-${district.address}]" } } else { newName2 = newName.replaceAll("_", "") + copyObj.name = newName2 //district = getMatchName(data.city, data.region, newName2, data.address,"district") - district = getMatchNameFromMap(data.city, data.region, newName2, data.address) + district = getMatchNameFromMap(copyObj) if (district) { type = 4; println "[匹配成功](All)[${district.city}-${district.region}-${district.name}-${district.address}]" @@ -410,15 +421,20 @@ class DataStandardBatch extends BaseBatch{ } if (!district) { - - district = getMatchNameFromMap(data.city, null, newName2, data.address) + copyObj.name = newName2 + copyObj.region = null + district = getMatchNameFromMap(copyObj) if(!district){ - district = getMatchNameFromMap(data.city, null, s1, data.address) + copyObj.name = s1 + copyObj.region = null + district = getMatchNameFromMap(copyObj) if (district) { type = 5; } else { if (s2 && s2.length() >= 3) { - district = getMatchNameFromMap(data.city, null, s2, data.address) + copyObj.name = s2 + copyObj.region = null + district = getMatchNameFromMap(copyObj) if (district) { type = 6; } @@ -446,11 +462,14 @@ class DataStandardBatch extends BaseBatch{ }catch (Exception e){ addr = data.address; } - district = getMatchAddressFromMap(data.city, data.region, addr, data.name) + def copyObj = data + copyObj.address = addr + district = getMatchAddressFromMap(copyObj) if(district){ type = 8; }else{ - district = getMatchAddressFromMap(data.city, null, addr, data.name) + copyObj.region = null + district = getMatchAddressFromMap(copyObj) if(district){ type = 9; } @@ -460,19 +479,23 @@ class DataStandardBatch extends BaseBatch{ if(!district){ def districts = getMatchPingYing(data.city, data.region, data.name, addr); - district = checkMatchAddress(districts, addr); + district = checkMatchAddress(districts, addr, data); if(district){ type = 10; }else if(districts.size()==1){ - district = districts[0]; - type = 14 + district = getFilerMatchDistrict(districts[0], data.estate_type, avg_price); + if(district){ + type = 14 + } } } if(!district){ - district = getOpenApiMatch(data.city, data.region, data.name, addr); + def copyObj = data + copyObj.address = addr + district = getOpenApiMatch(copyObj) if(!district && addr){ - district = getOpenApiMatch(data.city, data.region, addr, data.name); + district = getOpenApiMatch(copyObj) } if(district){ if(data.region != district.region){ @@ -522,7 +545,7 @@ class DataStandardBatch extends BaseBatch{ return null; } - def checkMatchAddress(def districts, def address){ + def checkMatchAddress(def districts, def address, def data){ def district = null; if(districts.size()==1 && address && address.matches(".+?(路|街|大道).+(号|弄)")){ def tempDist = districts[0]; @@ -537,6 +560,10 @@ class DataStandardBatch extends BaseBatch{ } } } + + if(district){ + return getFilerMatchDistrict(district, data.estate_type, data.avg_price) + } return district; } @@ -584,54 +611,152 @@ class DataStandardBatch extends BaseBatch{ }*/ - def getMatchNameFromMap(def city, def region, def disName, String address){ - //查询房价网是否存在该小区 - def dbo = findSameNameFromMap(city, region, disName); + def getMatchNameFromMap(def data){ + //查询房价网是否存在该小区 city, data.region, data.name, data.address, data.estate_type, avg_price + def dbo = findSameNameFromMap(data.city, data.region, data.name); def addrFlag = null - if(address){ - addrFlag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$') + if(data.address){ + addrFlag = data.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$') if(addrFlag){ - dbo = addressCheck(dbo, region, address); + dbo = addressCheck(dbo, data.region, data.address); } } if(!dbo){ //别名 - def distrs = findAliasNameFromMap(city, region, disName); + def distrs = findAliasNameFromMap(data.city, data.region, data.name); if(distrs && distrs.size()==1){ dbo = distrs[0] - }else if(distrs && distrs.size()>1 && address){ + }else if(distrs && distrs.size()>1 && data.address){ for(def d : distrs){ - if(addrFlag && (d.address.contains(address)||d.address==address || address.contains(d.address))){ + if(addrFlag && (d.address.contains(data.address)||d.address==data.address || data.address.contains(d.address))){ dbo = d; break; } } } if(dbo){ - dbo = addressCheck(dbo, region, address); + dbo = addressCheck(dbo, data.region, data.address); } //地址 if(!dbo){ - def result = findAddressFromMap(city, region, disName) + def result = findAddressFromMap(data.city, data.region, data.name) if(result.size()==1){ dbo = result[0]; - }else if(result && result.size()>1 && address){ + }else if(result && result.size()>1 && data.address){ for(def d : result){ - if(d.address.contains(address)||d.address==address || address.contains(d.address)){ + if(d.address.contains(data.address)||d.address==data.address || data.address.contains(d.address)){ dbo = d; break; } } } if(dbo){ - dbo = addressCheck(dbo, region, address); + dbo = addressCheck(dbo, data.region, data.address); } } } + if(dbo){ + return getFilerMatchDistrict(dbo, data) + } return dbo; } + def getFilerMatchDistrict(dbo, data){ + def types = [] + if(data.estate_type){ + types = getEstateTypes(data.estate_type) + } + if(types.size()>0 && types.size() == 1){ + if(types.contains(dbo.estate_type)){ + return dbo; + }else{ + def ret = THIRD_MATCH_FILTER_COLL.findOne(data._id) + if(!ret){ + data.put("msg","物业类型不匹配") + THIRD_MATCH_FILTER_COLL.save(data) + } + return null + } + }else{ + def priceMap = findHousePrice(dbo.city, dbo.region, dbo.name, DateUtils.format(DateUtil.add(new Date(), -180),"yyyyMMdd").toInteger()) + def price = priceMap.get(dbo.estate_type) + if(price){ + def pre = (data.avg_price-price)*100/price + if(pre>=-20 && pre<=20){ + return dbo; + }else{ + def ret = THIRD_PRICE_FILTER_COLL.findOne(data._id) + if(!ret){ + data.put("msg","房源价格过滤") + THIRD_PRICE_FILTER_COLL.save(data) + } + return null + } + } + } + return dbo + } + + + def findHousePrice(def city, def region, def name, def startDate){ + def typePriceMap = [:] + try{ + String sql = "select avg_price, estate_type from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and region='${region}' and district_name='${name}'" + def houseMap = [:] + ElasticUtils.executeSqlQuery(sql,{ data-> + if(data.estate_type){ + def key = "" + if(data.estate_type.contains("住宅") || data.estate_type.contains("公寓")){ + key = "普通住宅" + }else if(data.estate_type.contains("商住") || data.estate_type.contains("商务")){ + key = "商住" + }else if( data.estate_type.contains("别墅")){ + key = "别墅" + } + def houses = null + if(key){ + houses = houseMap.get(key) + if(!houses){ + houses = [] + } + } + if(data.avg_price){ + houses << data.avg_price + houseMap.put(key, houses) + } + } + }) + if(houseMap){ + houseMap.each{k,v-> + if(v && v.size()>=3){ + def price = (int)(v.sum()/v.size()) + typePriceMap.put(k, price) + } + } + } + }catch(Exception e){ + e.printStackTrace() + } + + return typePriceMap + } + + def getEstateTypes(str){ + def types = [] + if(str){ + if(str.contains("住宅") || str.contains("公寓")){ + types << "普通住宅" + } + if(str.contains("商住") || str.contains("商务")){ + types << "商住" + } + if(str.contains("别墅")){ + types << "别墅" + } + } + return types; + } /*def getMatchAddress(def city, def region, def address, def name, def cat){ def dbo = CommonDatautils.findSeaweed(city, region, address, cat); @@ -664,22 +789,23 @@ class DataStandardBatch extends BaseBatch{ } return dbo; }*/ - def getMatchAddressFromMap(def city, def region, def address, def name){ + def getMatchAddressFromMap(def copyObj){ def dbo = null - if(address && isStandardRoad(address)){ - dbo = findSameNameFromMap(city, region, address); + def name = copyObj.name + if(copyObj.address && isStandardRoad(copyObj.address)){ + dbo = findSameNameFromMap(copyObj.city, copyObj.region, copyObj.address); } if(!dbo){ def result = null - if(address && isStandardRoad(address)){ - result = findAddressFromMap(city, region, address); + if(copyObj.address && isStandardRoad(copyObj.address)){ + result = findAddressFromMap(copyObj.city, copyObj.region, copyObj.address); } if(result && result.size()==1){ dbo = result[0]; - }else if(result && result.size()>1 && name){ + }else if(result && result.size()>1 && copyObj.name){ def score = null; for(def d : result){ - if(d.name == name){ + if(d.name == copyObj.name){ dbo = d; break; }else{ @@ -706,7 +832,7 @@ class DataStandardBatch extends BaseBatch{ } } if(!dbo){ - result = findAddrAliasFromMap(city, region, address); + result = findAddrAliasFromMap(copyObj.city, copyObj.region, copyObj.address); if(result.size()==1){ dbo = result[0]; }else if(result && result.size()>1 && name){ @@ -718,6 +844,11 @@ class DataStandardBatch extends BaseBatch{ } } } + + if(dbo){ + return getFilerMatchDistrict(dbo, copyObj) + } + } return dbo; } @@ -756,7 +887,7 @@ class DataStandardBatch extends BaseBatch{ if(arrays[0] && arrays.size()==2){ def py2 = com.fangjia.core.util.DataUtil.hanyuQuanpin(arrays[0], " ") if(py1 == py2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] } } } @@ -768,11 +899,15 @@ class DataStandardBatch extends BaseBatch{ } - def getOpenApiMatch(def city, def region, def name, def address){ - if(StringUtils.isNotEmpty(name)){ - def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, null, "fuzzy", false) + def getOpenApiMatch(def data){ + if(StringUtils.isNotEmpty(data.name)){ + def jsonObject = OpenApiClient.addressMatch(data.city, data.name, "property", false, data.region, null, "fuzzy", false) if(jsonObject && jsonObject.credit>0 ){ - return [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address, credit:jsonObject.credit] + def dbo = [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address,estate_type:jsonObject.propertyType, credit:jsonObject.credit] + if(dbo){ + return getFilerMatchDistrict(dbo, data) + } + return dbo } } return null; @@ -783,6 +918,9 @@ class DataStandardBatch extends BaseBatch{ def findAllNoMatchData(){ DBObject qdata = new BasicDBObject(); qdata.put("s_date", SHORT_DATE) +// qdata.put("city","北京") + qdata.put("source","安居客") +// qdata.put("name",Pattern.compile("金隅澜湾.*")) qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false)); def ret = [] try{ @@ -948,9 +1086,9 @@ class DataStandardBatch extends BaseBatch{ def arrays = k.split("###") if(arrays[0] && arrays[0]==keyword){ if(arrays.size()>=2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:region, name:arrays[0]] + districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type] } } @@ -962,9 +1100,9 @@ class DataStandardBatch extends BaseBatch{ def arrays = k.split("###") if(arrays[0] && arrays[0]==keyword){ if(arrays.size()>=2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:region, name:arrays[0]] + districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type] } } } @@ -990,18 +1128,18 @@ class DataStandardBatch extends BaseBatch{ def alias = v.alias; if(alias.contains(keyword)){ if(arrays.size()>=2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:region, name:arrays[0]] + districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type] } } }else if(v && v.mapping_alias){ def mappingAlias = v.mapping_alias if(mappingAlias.contains(keyword)){ if(arrays.size()>=2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:region, name:arrays[0]] + districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type] } } } @@ -1016,18 +1154,18 @@ class DataStandardBatch extends BaseBatch{ def alias = v.alias; if(alias.contains(keyword)){ if(arrays.size()>=2){ - districts << [city:city, region:r, name:arrays[0],address:arrays[1]] + districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:r, name:arrays[0]] + districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type] } } }else if(v && v.mapping_alias){ def mappingAlias = v.mapping_alias if(mappingAlias.contains(keyword)){ if(arrays.size()>=2){ - districts << [city:city, region:r, name:arrays[0],address:arrays[1]] + districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:r, name:arrays[0]] + districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type] } } } @@ -1052,9 +1190,9 @@ class DataStandardBatch extends BaseBatch{ def alias = v.addr_alias; if(alias.contains(keyword)){ if(arrays.size()>=2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:region, name:arrays[0]] + districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type] } } } @@ -1068,9 +1206,9 @@ class DataStandardBatch extends BaseBatch{ def alias = v.addr_alias; if(alias.contains(keyword)){ if(arrays.size()>=2){ - districts << [city:city, region:r, name:arrays[0],address:arrays[1]] + districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:r, name:arrays[0]] + districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type] } } } @@ -1093,9 +1231,9 @@ class DataStandardBatch extends BaseBatch{ def arrays = k.split("###") if(arrays && arrays.size()==2 && arrays[1]==keyword){ if(arrays.size()>=2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:region, name:arrays[0]] + districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type] } } } @@ -1106,9 +1244,9 @@ class DataStandardBatch extends BaseBatch{ def arrays = k.split("###") if(arrays && arrays.size()==2 && arrays[1]==keyword){ if(arrays.size()>=2){ - districts << [city:city, region:r, name:arrays[0],address:arrays[1]] + districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:r, name:arrays[0]] + districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type] } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 68943db41..f954743e4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -24,8 +24,8 @@ class SynchDistrictPrice extends BaseBatch{ // "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"] def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"] - // "365淘房":"365淘房","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","吉屋":"吉屋" - def esSources = ["吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产"] + // "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产" + def esSources = ["中原地产":"中原地产"] static main(args) { @@ -57,7 +57,7 @@ class SynchDistrictPrice extends BaseBatch{ def names = new HashSet() def rets = [] ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>='${mdate}' order by m_date", {d-> - def s = d.city +"#"+d.region+"#"+d.name + def s = d.city +"#"+d.region+"#"+d.name+"#"+d.property_type if(!names.contains(s)){ names << s def data = new BasicDBObject() @@ -67,6 +67,7 @@ class SynchDistrictPrice extends BaseBatch{ data.put("address", d.address) data.put("avg_price", d.avg_price) data.put("source", mgSource) + data.put("estate_type",d.property_type) data.put("s_date", SHORT_DATE) data.put("url", d.url) data.put("c_date", new Date()) @@ -144,7 +145,7 @@ class SynchDistrictPrice extends BaseBatch{ println "["+source +"-"+city + "] add:"+ rets.size()+" 原抓取:"+count }else{ for(def r : rets){ - def obj= findIsExists(r.city, r.region, r.name, SHORT_DATE, source) + def obj= findIsExists(r.city, r.region, r.name, r.estate_type, SHORT_DATE, source) if(obj){ updateIndex++ // thirdPartyColl.remove(new BasicDBObject('_id',obj._id)) @@ -152,6 +153,7 @@ class SynchDistrictPrice extends BaseBatch{ def udata = new BasicDBObject() udata.put("avg_price", r.avg_price) + udata.put("estate_type", r.estate_type) udata.put("m_date",new Date()) thirdPartyColl.update(new BasicDBObject('_id',obj._id), new BasicDBObject('$set', udata)) @@ -181,13 +183,16 @@ class SynchDistrictPrice extends BaseBatch{ } - def findIsExists(def city, def region, def name, def sDate, def source){ + def findIsExists(def city, def region, def name, estate_type, def sDate, def source){ def q = new BasicDBObject() q.put("source", source) q.put("s_date", sDate) q.put("city", city) q.put("region", region) q.put("name", name) + if(estate_type){ + q.put("estate_type",estate_type) + } def obj = thirdPartyColl.findOne(q) return obj }