diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 425f25225..c06cef98e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -35,7 +35,7 @@ class BigdataCommunityBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = ["临夏"]//CcbCommonUtils.citys_all; + def citys = CcbCommonUtils.citys_92; def rows = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 573d65d6c..ca517f31a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -75,8 +75,8 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); -// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220818, 20220919); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220918, 20221019); +// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); //测试通过匹配接口配房源效果 // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 50e24d4fd..c673347dd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -16,13 +16,15 @@ class ExportOracleData { "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") def cityCodeMap = CcbCommonUtils.findCityCodeValue() + def citys = null static main(args) { def task = new ExportOracleData() - task.run() + task.run(args) } - def run() { + def run(args) { + citys = CcbCommonUtils.getRunningCitys(args) // def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") @@ -131,7 +133,6 @@ class ExportOracleData { def exportBigdataPoiData() { - def citys = CcbCommonUtils.citys_all def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) @@ -201,7 +202,6 @@ class ExportOracleData { // def citys = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"] // def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def allRows = [] - def citys = CcbCommonUtils.citys_all for(def city : citys) { def cityCode = cityCodeMap.get(city) def sql = "select CITYID, COMMID, BUILDTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, WORKDATE from bigdata_commbuildtype_gj where CITYID='${cityCode}'"; @@ -234,7 +234,6 @@ class ExportOracleData { } def exportBigdataDistrictData() { - def citys = CcbCommonUtils.citys_all def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) @@ -276,7 +275,6 @@ class ExportOracleData { def exportGuidedPrice() { - def citys = CcbCommonUtils.citys_all def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) @@ -315,7 +313,6 @@ class ExportOracleData { } def exportCommunityGuidedPrice(def date) { - def citys = CcbCommonUtils.citys_all def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) @@ -355,7 +352,6 @@ class ExportOracleData { def exportBigdataDeal(def dealDate, def new_dists) { def allRows = [] - def citys = CcbCommonUtils.citys_all for(def city : citys) { def cityCode = cityCodeMap.get(city) if(new_dists){ @@ -531,7 +527,6 @@ class ExportOracleData { } def headers = [] def index = 0 - def citys = CcbCommonUtils.citys_all for(def city : citys) { def cityCode = cityCodeMap.get(city) def allRows = [] @@ -589,7 +584,6 @@ class ExportOracleData { def exportBigdataCommunity(){ - def citys = CcbCommonUtils.citys_all def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) @@ -649,8 +643,6 @@ class ExportOracleData { def exportBigdataForsale(publishDate, new_dists){ //["北京","上海"] - def citys = CcbCommonUtils.citys_all; - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index ec63b0189..6b1009e6c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -42,7 +42,7 @@ class ThirdPartyPriceBatch extends BaseBatch { @Override public void process(Object args) { - def citys = CcbCommonUtils.citys_252; + def citys = CcbCommonUtils.getRunningCitys(args) def rows = [] def index = 0 def weekend = DateUtils.getFirstDayOfWeek(new Date()) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index b2bbe9713..c2f488b5f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -73,15 +73,15 @@ class TradeStatLog { } def run(){ - def citys = CcbCommonUtils.citys; + def citys = CcbCommonUtils.citys_92; def rets = [] - rets << ["城市","6月总量","7月总量","8月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] - def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"] + rets << ["城市","7月总量","8月总量","9月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] + def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"] //def startDate = "2021-05-01" - def startDate = "2022-06-01" - def date = "2022-07-01" - def endDate = "2022-08-01" - def newDate = "2022-09-01" + def startDate = "2022-07-01" + def date = "2022-08-01" + def endDate = "2022-09-01" + def newDate = "2022-10-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index e5281ae53..3be6ffa9a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -473,6 +473,7 @@ class CommonDatautils { if(dbo.mapping_alias){ detailMap.mapping_alias = dbo.mapping_alias; } + detailMap.estate_type = dbo.estate_type2 } }catch(Exception e){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index 2aed5a7f1..6ccaabde3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -54,7 +54,7 @@ class YouDaDataParser { // task.newHouseparse();//友达一手房存入mongo // task.esfParser("D:\\数据\\友达\\2022年10月\\查询1.DBF");//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es - task.esfhouseToSaveEs();//二手房保存到es +// task.esfhouseToSaveEs();//二手房保存到es // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index 9a9fd90ad..91d3dee7e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.thirdPriceCrawl +import com.fangjia.core.util.DateUtil import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher @@ -11,7 +12,7 @@ import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.CosineSimilarAlgorithm import com.fangjia.script.util.DataUtil import com.fangjia.script.util.DbUtils - +import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject @@ -77,6 +78,9 @@ class DataStandardBatch extends BaseBatch{ } def rowIndex = 0; + + DBCollection THIRD_MATCH_FILTER_COLL = MongoUtils.getCollectionByName("fangjia_craw", "third_match_filter", "192.168.0.235", 27018) + DBCollection THIRD_PRICE_FILTER_COLL = MongoUtils.getCollectionByName("fangjia_craw", "third_price_filter", "192.168.0.235", 27018) static main(args) { runBatch(new DataStandardBatch(), args) } @@ -87,9 +91,11 @@ class DataStandardBatch extends BaseBatch{ // ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl) cityMap.each{city,alias-> +// if(city == "北京"){ println "开始加载【${city}】" - def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias"]) + def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"]) districtMap.putAll(detailMap) +// } } while(flag){ @@ -370,39 +376,44 @@ class DataStandardBatch extends BaseBatch{ if(data.name.contains("写字楼")||data.name.contains("办公楼")||data.name.contains("商业")){ district = null; }else{ - district = getMatchNameFromMap(data.city, data.region, data.name, data.address) + district = getMatchNameFromMap(data) if(!district && data.name.length()==2){//没有匹配上,小区名只有2个字就把小区名加“小区” - district = getMatchNameFromMap(data.city, data.region, data.name+"小区", data.address) + def copyObj = data + copyObj.name = data.name+"小区" + district = getMatchNameFromMap(copyObj) } - if(!district){ def newName = filterSpacialChar(data.name);//包含特殊符号 if(newName) { - district = getMatchNameFromMap(data.city, data.region, newName.replaceAll("_", ""), data.address) + def copyObj = data + copyObj.name = newName.replaceAll("_", "") + district = getMatchNameFromMap(copyObj) if(!district){ def s1 = newName.substring(0, newName.indexOf("_")) if (s1) { //特殊符号前的内容匹配 //district = getMatchName(data.city, data.region, s1, data.address,"district") - district = getMatchNameFromMap(data.city, data.region, s1, data.address) + copyObj.name = s1 + district = getMatchNameFromMap(copyObj) if (!district) { - def s2 = newName.replace(s1, "").replaceAll("_", "") def newName2 = null; if (s2 && s2.length() >= 3) { //特殊符号后的内容匹配 // district = getMatchName(data.city, data.region, s2, data.address,"district") - district = getMatchNameFromMap(data.city, data.region, s2, data.address) + copyObj.name = s2 + district = getMatchNameFromMap(copyObj) if (district) { type = 3; println "[匹配成功](后)[${district.city}-${district.region}-${district.name}-${district.address}]" } } else { newName2 = newName.replaceAll("_", "") + copyObj.name = newName2 //district = getMatchName(data.city, data.region, newName2, data.address,"district") - district = getMatchNameFromMap(data.city, data.region, newName2, data.address) + district = getMatchNameFromMap(copyObj) if (district) { type = 4; println "[匹配成功](All)[${district.city}-${district.region}-${district.name}-${district.address}]" @@ -410,15 +421,20 @@ class DataStandardBatch extends BaseBatch{ } if (!district) { - - district = getMatchNameFromMap(data.city, null, newName2, data.address) + copyObj.name = newName2 + copyObj.region = null + district = getMatchNameFromMap(copyObj) if(!district){ - district = getMatchNameFromMap(data.city, null, s1, data.address) + copyObj.name = s1 + copyObj.region = null + district = getMatchNameFromMap(copyObj) if (district) { type = 5; } else { if (s2 && s2.length() >= 3) { - district = getMatchNameFromMap(data.city, null, s2, data.address) + copyObj.name = s2 + copyObj.region = null + district = getMatchNameFromMap(copyObj) if (district) { type = 6; } @@ -446,11 +462,14 @@ class DataStandardBatch extends BaseBatch{ }catch (Exception e){ addr = data.address; } - district = getMatchAddressFromMap(data.city, data.region, addr, data.name) + def copyObj = data + copyObj.address = addr + district = getMatchAddressFromMap(copyObj) if(district){ type = 8; }else{ - district = getMatchAddressFromMap(data.city, null, addr, data.name) + copyObj.region = null + district = getMatchAddressFromMap(copyObj) if(district){ type = 9; } @@ -460,19 +479,23 @@ class DataStandardBatch extends BaseBatch{ if(!district){ def districts = getMatchPingYing(data.city, data.region, data.name, addr); - district = checkMatchAddress(districts, addr); + district = checkMatchAddress(districts, addr, data); if(district){ type = 10; }else if(districts.size()==1){ - district = districts[0]; - type = 14 + district = getFilerMatchDistrict(districts[0], data); + if(district){ + type = 14 + } } } if(!district){ - district = getOpenApiMatch(data.city, data.region, data.name, addr); + def copyObj = data + copyObj.address = addr + district = getOpenApiMatch(copyObj) if(!district && addr){ - district = getOpenApiMatch(data.city, data.region, addr, data.name); + district = getOpenApiMatch(copyObj) } if(district){ if(data.region != district.region){ @@ -522,7 +545,7 @@ class DataStandardBatch extends BaseBatch{ return null; } - def checkMatchAddress(def districts, def address){ + def checkMatchAddress(def districts, def address, def data){ def district = null; if(districts.size()==1 && address && address.matches(".+?(路|街|大道).+(号|弄)")){ def tempDist = districts[0]; @@ -537,6 +560,10 @@ class DataStandardBatch extends BaseBatch{ } } } + + if(district){ + return getFilerMatchDistrict(district, data) + } return district; } @@ -584,54 +611,154 @@ class DataStandardBatch extends BaseBatch{ }*/ - def getMatchNameFromMap(def city, def region, def disName, String address){ - //查询房价网是否存在该小区 - def dbo = findSameNameFromMap(city, region, disName); + def getMatchNameFromMap(def data){ + //查询房价网是否存在该小区 city, data.region, data.name, data.address, data.estate_type, avg_price + def dbo = findSameNameFromMap(data.city, data.region, data.name); def addrFlag = null - if(address){ - addrFlag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$') + if(data.address){ + addrFlag = data.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$') if(addrFlag){ - dbo = addressCheck(dbo, region, address); + dbo = addressCheck(dbo, data.region, data.address); } } if(!dbo){ //别名 - def distrs = findAliasNameFromMap(city, region, disName); + def distrs = findAliasNameFromMap(data.city, data.region, data.name); if(distrs && distrs.size()==1){ dbo = distrs[0] - }else if(distrs && distrs.size()>1 && address){ + }else if(distrs && distrs.size()>1 && data.address){ for(def d : distrs){ - if(addrFlag && (d.address.contains(address)||d.address==address || address.contains(d.address))){ + if(addrFlag && (d.address.contains(data.address)||d.address==data.address || data.address.contains(d.address))){ dbo = d; break; } } } if(dbo){ - dbo = addressCheck(dbo, region, address); + dbo = addressCheck(dbo, data.region, data.address); } //地址 if(!dbo){ - def result = findAddressFromMap(city, region, disName) + def result = findAddressFromMap(data.city, data.region, data.name) if(result.size()==1){ dbo = result[0]; - }else if(result && result.size()>1 && address){ + }else if(result && result.size()>1 && data.address){ for(def d : result){ - if(d.address.contains(address)||d.address==address || address.contains(d.address)){ + if(d.address.contains(data.address)||d.address==data.address || data.address.contains(d.address)){ dbo = d; break; } } } if(dbo){ - dbo = addressCheck(dbo, region, address); + dbo = addressCheck(dbo, data.region, data.address); } } } + if(dbo){ + return getFilerMatchDistrict(dbo, data) + } return dbo; } + def getFilerMatchDistrict(dbo, data){ + def types = [] + if(data.estate_type){ + types = getEstateTypes(data.estate_type) + } + if(types.size()>0 && types.size() == 1){ + if(types.contains(dbo.estate_type)){ + return dbo; + }else{ + def ret = THIRD_MATCH_FILTER_COLL.findOne(data._id) + if(!ret){ + data.put("msg","物业类型不匹配") + THIRD_MATCH_FILTER_COLL.save(data) + } + return null + } + }else{ + def priceMap = findHousePrice(dbo.city, dbo.region, dbo.name, DateUtils.format(DateUtil.add(new Date(), -180),"yyyyMMdd").toInteger()) + def price = priceMap.get(dbo.estate_type) + if(price && data.avg_price && data.avg_price.toString().isNumber()){ + def pre = (data.avg_price-price)*100/price + if(pre>=-20 && pre<=20){ + return dbo; + }else{ + def ret = THIRD_PRICE_FILTER_COLL.findOne(data._id) + if(!ret){ + data.put("msg","房源价格过滤") + THIRD_PRICE_FILTER_COLL.save(data) + } + return null + } + } + } + return dbo + } + + + def findHousePrice(def city, def region, def name, def startDate){ + def typePriceMap = [:] + try{ + String sql = "select avg_price, estate_type from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and region='${region}' and district_name='${name}'" + def houseMap = [:] + ElasticUtils.executeSqlQuery(sql,{ data-> + if(data.estate_type){ + def key = "" + if(data.estate_type.contains("住宅") || data.estate_type.contains("公寓")){ + key = "普通住宅" + }else if(data.estate_type.contains("商住") || data.estate_type.contains("商务")){ + key = "商住" + }else if( data.estate_type.contains("别墅")){ + key = "别墅" + }else{ + key = "其它" + } + def houses = null + if(key){ + houses = houseMap.get(key) + if(!houses){ + houses = [] + } + } + if(data.avg_price){ + houses << data.avg_price + houseMap.put(key, houses) + } + } + }) + if(houseMap){ + houseMap.each{k,v-> + if(v && v.size()>=3){ + def price = (int)(v.sum()/v.size()) + typePriceMap.put(k, price) + } + } + } + }catch(Exception e){ + e.printStackTrace() + } + + return typePriceMap + } + + def getEstateTypes(str){ + def types = [] + if(str){ + if(str.contains("住宅") || str.contains("公寓")){ + types << "普通住宅" + } + if(str.contains("商住") || str.contains("商务")){ + types << "商住" + } + if(str.contains("别墅")){ + types << "别墅" + } + } + return types; + } /*def getMatchAddress(def city, def region, def address, def name, def cat){ def dbo = CommonDatautils.findSeaweed(city, region, address, cat); @@ -664,22 +791,23 @@ class DataStandardBatch extends BaseBatch{ } return dbo; }*/ - def getMatchAddressFromMap(def city, def region, def address, def name){ + def getMatchAddressFromMap(def copyObj){ def dbo = null - if(address && isStandardRoad(address)){ - dbo = findSameNameFromMap(city, region, address); + def name = copyObj.name + if(copyObj.address && isStandardRoad(copyObj.address)){ + dbo = findSameNameFromMap(copyObj.city, copyObj.region, copyObj.address); } if(!dbo){ def result = null - if(address && isStandardRoad(address)){ - result = findAddressFromMap(city, region, address); + if(copyObj.address && isStandardRoad(copyObj.address)){ + result = findAddressFromMap(copyObj.city, copyObj.region, copyObj.address); } if(result && result.size()==1){ dbo = result[0]; - }else if(result && result.size()>1 && name){ + }else if(result && result.size()>1 && copyObj.name){ def score = null; for(def d : result){ - if(d.name == name){ + if(d.name == copyObj.name){ dbo = d; break; }else{ @@ -706,7 +834,7 @@ class DataStandardBatch extends BaseBatch{ } } if(!dbo){ - result = findAddrAliasFromMap(city, region, address); + result = findAddrAliasFromMap(copyObj.city, copyObj.region, copyObj.address); if(result.size()==1){ dbo = result[0]; }else if(result && result.size()>1 && name){ @@ -718,6 +846,11 @@ class DataStandardBatch extends BaseBatch{ } } } + + if(dbo){ + return getFilerMatchDistrict(dbo, copyObj) + } + } return dbo; } @@ -756,7 +889,7 @@ class DataStandardBatch extends BaseBatch{ if(arrays[0] && arrays.size()==2){ def py2 = com.fangjia.core.util.DataUtil.hanyuQuanpin(arrays[0], " ") if(py1 == py2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] } } } @@ -768,11 +901,15 @@ class DataStandardBatch extends BaseBatch{ } - def getOpenApiMatch(def city, def region, def name, def address){ - if(StringUtils.isNotEmpty(name)){ - def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, null, "fuzzy", false) + def getOpenApiMatch(def data){ + if(StringUtils.isNotEmpty(data.name)){ + def jsonObject = OpenApiClient.addressMatch(data.city, data.name, "property", false, data.region, null, "fuzzy", false) if(jsonObject && jsonObject.credit>0 ){ - return [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address, credit:jsonObject.credit] + def dbo = [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address,estate_type:jsonObject.propertyType, credit:jsonObject.credit] + if(dbo){ + return getFilerMatchDistrict(dbo, data) + } + return dbo } } return null; @@ -783,6 +920,9 @@ class DataStandardBatch extends BaseBatch{ def findAllNoMatchData(){ DBObject qdata = new BasicDBObject(); qdata.put("s_date", SHORT_DATE) +// qdata.put("city","北京") +// qdata.put("source","安居客") +// qdata.put("name",Pattern.compile("金隅澜湾.*")) qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false)); def ret = [] try{ @@ -948,9 +1088,9 @@ class DataStandardBatch extends BaseBatch{ def arrays = k.split("###") if(arrays[0] && arrays[0]==keyword){ if(arrays.size()>=2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:region, name:arrays[0]] + districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type] } } @@ -962,9 +1102,9 @@ class DataStandardBatch extends BaseBatch{ def arrays = k.split("###") if(arrays[0] && arrays[0]==keyword){ if(arrays.size()>=2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:region, name:arrays[0]] + districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type] } } } @@ -990,18 +1130,18 @@ class DataStandardBatch extends BaseBatch{ def alias = v.alias; if(alias.contains(keyword)){ if(arrays.size()>=2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:region, name:arrays[0]] + districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type] } } }else if(v && v.mapping_alias){ def mappingAlias = v.mapping_alias if(mappingAlias.contains(keyword)){ if(arrays.size()>=2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:region, name:arrays[0]] + districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type] } } } @@ -1016,18 +1156,18 @@ class DataStandardBatch extends BaseBatch{ def alias = v.alias; if(alias.contains(keyword)){ if(arrays.size()>=2){ - districts << [city:city, region:r, name:arrays[0],address:arrays[1]] + districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:r, name:arrays[0]] + districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type] } } }else if(v && v.mapping_alias){ def mappingAlias = v.mapping_alias if(mappingAlias.contains(keyword)){ if(arrays.size()>=2){ - districts << [city:city, region:r, name:arrays[0],address:arrays[1]] + districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:r, name:arrays[0]] + districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type] } } } @@ -1052,9 +1192,9 @@ class DataStandardBatch extends BaseBatch{ def alias = v.addr_alias; if(alias.contains(keyword)){ if(arrays.size()>=2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:region, name:arrays[0]] + districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type] } } } @@ -1068,9 +1208,9 @@ class DataStandardBatch extends BaseBatch{ def alias = v.addr_alias; if(alias.contains(keyword)){ if(arrays.size()>=2){ - districts << [city:city, region:r, name:arrays[0],address:arrays[1]] + districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:r, name:arrays[0]] + districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type] } } } @@ -1093,9 +1233,9 @@ class DataStandardBatch extends BaseBatch{ def arrays = k.split("###") if(arrays && arrays.size()==2 && arrays[1]==keyword){ if(arrays.size()>=2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:region, name:arrays[0]] + districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type] } } } @@ -1106,9 +1246,9 @@ class DataStandardBatch extends BaseBatch{ def arrays = k.split("###") if(arrays && arrays.size()==2 && arrays[1]==keyword){ if(arrays.size()>=2){ - districts << [city:city, region:r, name:arrays[0],address:arrays[1]] + districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:r, name:arrays[0]] + districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type] } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 68943db41..f954743e4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -24,8 +24,8 @@ class SynchDistrictPrice extends BaseBatch{ // "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"] def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"] - // "365淘房":"365淘房","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","吉屋":"吉屋" - def esSources = ["吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产"] + // "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产" + def esSources = ["中原地产":"中原地产"] static main(args) { @@ -57,7 +57,7 @@ class SynchDistrictPrice extends BaseBatch{ def names = new HashSet() def rets = [] ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>='${mdate}' order by m_date", {d-> - def s = d.city +"#"+d.region+"#"+d.name + def s = d.city +"#"+d.region+"#"+d.name+"#"+d.property_type if(!names.contains(s)){ names << s def data = new BasicDBObject() @@ -67,6 +67,7 @@ class SynchDistrictPrice extends BaseBatch{ data.put("address", d.address) data.put("avg_price", d.avg_price) data.put("source", mgSource) + data.put("estate_type",d.property_type) data.put("s_date", SHORT_DATE) data.put("url", d.url) data.put("c_date", new Date()) @@ -144,7 +145,7 @@ class SynchDistrictPrice extends BaseBatch{ println "["+source +"-"+city + "] add:"+ rets.size()+" 原抓取:"+count }else{ for(def r : rets){ - def obj= findIsExists(r.city, r.region, r.name, SHORT_DATE, source) + def obj= findIsExists(r.city, r.region, r.name, r.estate_type, SHORT_DATE, source) if(obj){ updateIndex++ // thirdPartyColl.remove(new BasicDBObject('_id',obj._id)) @@ -152,6 +153,7 @@ class SynchDistrictPrice extends BaseBatch{ def udata = new BasicDBObject() udata.put("avg_price", r.avg_price) + udata.put("estate_type", r.estate_type) udata.put("m_date",new Date()) thirdPartyColl.update(new BasicDBObject('_id',obj._id), new BasicDBObject('$set', udata)) @@ -181,13 +183,16 @@ class SynchDistrictPrice extends BaseBatch{ } - def findIsExists(def city, def region, def name, def sDate, def source){ + def findIsExists(def city, def region, def name, estate_type, def sDate, def source){ def q = new BasicDBObject() q.put("source", source) q.put("s_date", sDate) q.put("city", city) q.put("region", region) q.put("name", name) + if(estate_type){ + q.put("estate_type",estate_type) + } def obj = thirdPartyColl.findOne(q) return obj }