From 7b5619f63baa13efae6eee3444323f34e93c384b Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 19 Oct 2022 10:13:22 +0800 Subject: [PATCH 1/6] =?UTF-8?q?=E5=BB=BA=E8=A1=8C=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../BigdataCommunityBatch.groovy | 2 +- .../ccbevaluation/BigdataForsaleBatch.groovy | 4 +- .../script/commData/CommonDatautils.groovy | 1 + .../thirdPriceCrawl/DataStandardBatch.groovy | 276 +++++++++++++----- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 15 +- 5 files changed, 221 insertions(+), 77 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 425f25225..c06cef98e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -35,7 +35,7 @@ class BigdataCommunityBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = ["临夏"]//CcbCommonUtils.citys_all; + def citys = CcbCommonUtils.citys_92; def rows = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 95ec61b0d..6ed03664a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -74,8 +74,8 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); -// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220818, 20220919); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220918, 20221019); +// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); //测试通过匹配接口配房源效果 // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index e5281ae53..3be6ffa9a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -473,6 +473,7 @@ class CommonDatautils { if(dbo.mapping_alias){ detailMap.mapping_alias = dbo.mapping_alias; } + detailMap.estate_type = dbo.estate_type2 } }catch(Exception e){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index 9a9fd90ad..721bf7025 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.thirdPriceCrawl +import com.fangjia.core.util.DateUtil import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher @@ -11,7 +12,7 @@ import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.CosineSimilarAlgorithm import com.fangjia.script.util.DataUtil import com.fangjia.script.util.DbUtils - +import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject @@ -77,6 +78,9 @@ class DataStandardBatch extends BaseBatch{ } def rowIndex = 0; + + DBCollection THIRD_MATCH_FILTER_COLL = MongoUtils.getCollectionByName("fangjia_craw", "third_match_filter", "192.168.0.235", 27018) + DBCollection THIRD_PRICE_FILTER_COLL = MongoUtils.getCollectionByName("fangjia_craw", "third_price_filter", "192.168.0.235", 27018) static main(args) { runBatch(new DataStandardBatch(), args) } @@ -87,9 +91,11 @@ class DataStandardBatch extends BaseBatch{ // ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl) cityMap.each{city,alias-> +// if(city == "北京"){ println "开始加载【${city}】" - def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias"]) + def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"]) districtMap.putAll(detailMap) +// } } while(flag){ @@ -370,39 +376,44 @@ class DataStandardBatch extends BaseBatch{ if(data.name.contains("写字楼")||data.name.contains("办公楼")||data.name.contains("商业")){ district = null; }else{ - district = getMatchNameFromMap(data.city, data.region, data.name, data.address) + district = getMatchNameFromMap(data) if(!district && data.name.length()==2){//没有匹配上,小区名只有2个字就把小区名加“小区” - district = getMatchNameFromMap(data.city, data.region, data.name+"小区", data.address) + def copyObj = data + copyObj.name = data.name+"小区" + district = getMatchNameFromMap(copyObj) } - if(!district){ def newName = filterSpacialChar(data.name);//包含特殊符号 if(newName) { - district = getMatchNameFromMap(data.city, data.region, newName.replaceAll("_", ""), data.address) + def copyObj = data + copyObj.name = newName.replaceAll("_", "") + district = getMatchNameFromMap(copyObj) if(!district){ def s1 = newName.substring(0, newName.indexOf("_")) if (s1) { //特殊符号前的内容匹配 //district = getMatchName(data.city, data.region, s1, data.address,"district") - district = getMatchNameFromMap(data.city, data.region, s1, data.address) + copyObj.name = s1 + district = getMatchNameFromMap(copyObj) if (!district) { - def s2 = newName.replace(s1, "").replaceAll("_", "") def newName2 = null; if (s2 && s2.length() >= 3) { //特殊符号后的内容匹配 // district = getMatchName(data.city, data.region, s2, data.address,"district") - district = getMatchNameFromMap(data.city, data.region, s2, data.address) + copyObj.name = s2 + district = getMatchNameFromMap(copyObj) if (district) { type = 3; println "[匹配成功](后)[${district.city}-${district.region}-${district.name}-${district.address}]" } } else { newName2 = newName.replaceAll("_", "") + copyObj.name = newName2 //district = getMatchName(data.city, data.region, newName2, data.address,"district") - district = getMatchNameFromMap(data.city, data.region, newName2, data.address) + district = getMatchNameFromMap(copyObj) if (district) { type = 4; println "[匹配成功](All)[${district.city}-${district.region}-${district.name}-${district.address}]" @@ -410,15 +421,20 @@ class DataStandardBatch extends BaseBatch{ } if (!district) { - - district = getMatchNameFromMap(data.city, null, newName2, data.address) + copyObj.name = newName2 + copyObj.region = null + district = getMatchNameFromMap(copyObj) if(!district){ - district = getMatchNameFromMap(data.city, null, s1, data.address) + copyObj.name = s1 + copyObj.region = null + district = getMatchNameFromMap(copyObj) if (district) { type = 5; } else { if (s2 && s2.length() >= 3) { - district = getMatchNameFromMap(data.city, null, s2, data.address) + copyObj.name = s2 + copyObj.region = null + district = getMatchNameFromMap(copyObj) if (district) { type = 6; } @@ -446,11 +462,14 @@ class DataStandardBatch extends BaseBatch{ }catch (Exception e){ addr = data.address; } - district = getMatchAddressFromMap(data.city, data.region, addr, data.name) + def copyObj = data + copyObj.address = addr + district = getMatchAddressFromMap(copyObj) if(district){ type = 8; }else{ - district = getMatchAddressFromMap(data.city, null, addr, data.name) + copyObj.region = null + district = getMatchAddressFromMap(copyObj) if(district){ type = 9; } @@ -460,19 +479,23 @@ class DataStandardBatch extends BaseBatch{ if(!district){ def districts = getMatchPingYing(data.city, data.region, data.name, addr); - district = checkMatchAddress(districts, addr); + district = checkMatchAddress(districts, addr, data); if(district){ type = 10; }else if(districts.size()==1){ - district = districts[0]; - type = 14 + district = getFilerMatchDistrict(districts[0], data.estate_type, avg_price); + if(district){ + type = 14 + } } } if(!district){ - district = getOpenApiMatch(data.city, data.region, data.name, addr); + def copyObj = data + copyObj.address = addr + district = getOpenApiMatch(copyObj) if(!district && addr){ - district = getOpenApiMatch(data.city, data.region, addr, data.name); + district = getOpenApiMatch(copyObj) } if(district){ if(data.region != district.region){ @@ -522,7 +545,7 @@ class DataStandardBatch extends BaseBatch{ return null; } - def checkMatchAddress(def districts, def address){ + def checkMatchAddress(def districts, def address, def data){ def district = null; if(districts.size()==1 && address && address.matches(".+?(路|街|大道).+(号|弄)")){ def tempDist = districts[0]; @@ -537,6 +560,10 @@ class DataStandardBatch extends BaseBatch{ } } } + + if(district){ + return getFilerMatchDistrict(district, data.estate_type, data.avg_price) + } return district; } @@ -584,54 +611,152 @@ class DataStandardBatch extends BaseBatch{ }*/ - def getMatchNameFromMap(def city, def region, def disName, String address){ - //查询房价网是否存在该小区 - def dbo = findSameNameFromMap(city, region, disName); + def getMatchNameFromMap(def data){ + //查询房价网是否存在该小区 city, data.region, data.name, data.address, data.estate_type, avg_price + def dbo = findSameNameFromMap(data.city, data.region, data.name); def addrFlag = null - if(address){ - addrFlag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$') + if(data.address){ + addrFlag = data.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$') if(addrFlag){ - dbo = addressCheck(dbo, region, address); + dbo = addressCheck(dbo, data.region, data.address); } } if(!dbo){ //别名 - def distrs = findAliasNameFromMap(city, region, disName); + def distrs = findAliasNameFromMap(data.city, data.region, data.name); if(distrs && distrs.size()==1){ dbo = distrs[0] - }else if(distrs && distrs.size()>1 && address){ + }else if(distrs && distrs.size()>1 && data.address){ for(def d : distrs){ - if(addrFlag && (d.address.contains(address)||d.address==address || address.contains(d.address))){ + if(addrFlag && (d.address.contains(data.address)||d.address==data.address || data.address.contains(d.address))){ dbo = d; break; } } } if(dbo){ - dbo = addressCheck(dbo, region, address); + dbo = addressCheck(dbo, data.region, data.address); } //地址 if(!dbo){ - def result = findAddressFromMap(city, region, disName) + def result = findAddressFromMap(data.city, data.region, data.name) if(result.size()==1){ dbo = result[0]; - }else if(result && result.size()>1 && address){ + }else if(result && result.size()>1 && data.address){ for(def d : result){ - if(d.address.contains(address)||d.address==address || address.contains(d.address)){ + if(d.address.contains(data.address)||d.address==data.address || data.address.contains(d.address)){ dbo = d; break; } } } if(dbo){ - dbo = addressCheck(dbo, region, address); + dbo = addressCheck(dbo, data.region, data.address); } } } + if(dbo){ + return getFilerMatchDistrict(dbo, data) + } return dbo; } + def getFilerMatchDistrict(dbo, data){ + def types = [] + if(data.estate_type){ + types = getEstateTypes(data.estate_type) + } + if(types.size()>0 && types.size() == 1){ + if(types.contains(dbo.estate_type)){ + return dbo; + }else{ + def ret = THIRD_MATCH_FILTER_COLL.findOne(data._id) + if(!ret){ + data.put("msg","物业类型不匹配") + THIRD_MATCH_FILTER_COLL.save(data) + } + return null + } + }else{ + def priceMap = findHousePrice(dbo.city, dbo.region, dbo.name, DateUtils.format(DateUtil.add(new Date(), -180),"yyyyMMdd").toInteger()) + def price = priceMap.get(dbo.estate_type) + if(price){ + def pre = (data.avg_price-price)*100/price + if(pre>=-20 && pre<=20){ + return dbo; + }else{ + def ret = THIRD_PRICE_FILTER_COLL.findOne(data._id) + if(!ret){ + data.put("msg","房源价格过滤") + THIRD_PRICE_FILTER_COLL.save(data) + } + return null + } + } + } + return dbo + } + + + def findHousePrice(def city, def region, def name, def startDate){ + def typePriceMap = [:] + try{ + String sql = "select avg_price, estate_type from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and region='${region}' and district_name='${name}'" + def houseMap = [:] + ElasticUtils.executeSqlQuery(sql,{ data-> + if(data.estate_type){ + def key = "" + if(data.estate_type.contains("住宅") || data.estate_type.contains("公寓")){ + key = "普通住宅" + }else if(data.estate_type.contains("商住") || data.estate_type.contains("商务")){ + key = "商住" + }else if( data.estate_type.contains("别墅")){ + key = "别墅" + } + def houses = null + if(key){ + houses = houseMap.get(key) + if(!houses){ + houses = [] + } + } + if(data.avg_price){ + houses << data.avg_price + houseMap.put(key, houses) + } + } + }) + if(houseMap){ + houseMap.each{k,v-> + if(v && v.size()>=3){ + def price = (int)(v.sum()/v.size()) + typePriceMap.put(k, price) + } + } + } + }catch(Exception e){ + e.printStackTrace() + } + + return typePriceMap + } + + def getEstateTypes(str){ + def types = [] + if(str){ + if(str.contains("住宅") || str.contains("公寓")){ + types << "普通住宅" + } + if(str.contains("商住") || str.contains("商务")){ + types << "商住" + } + if(str.contains("别墅")){ + types << "别墅" + } + } + return types; + } /*def getMatchAddress(def city, def region, def address, def name, def cat){ def dbo = CommonDatautils.findSeaweed(city, region, address, cat); @@ -664,22 +789,23 @@ class DataStandardBatch extends BaseBatch{ } return dbo; }*/ - def getMatchAddressFromMap(def city, def region, def address, def name){ + def getMatchAddressFromMap(def copyObj){ def dbo = null - if(address && isStandardRoad(address)){ - dbo = findSameNameFromMap(city, region, address); + def name = copyObj.name + if(copyObj.address && isStandardRoad(copyObj.address)){ + dbo = findSameNameFromMap(copyObj.city, copyObj.region, copyObj.address); } if(!dbo){ def result = null - if(address && isStandardRoad(address)){ - result = findAddressFromMap(city, region, address); + if(copyObj.address && isStandardRoad(copyObj.address)){ + result = findAddressFromMap(copyObj.city, copyObj.region, copyObj.address); } if(result && result.size()==1){ dbo = result[0]; - }else if(result && result.size()>1 && name){ + }else if(result && result.size()>1 && copyObj.name){ def score = null; for(def d : result){ - if(d.name == name){ + if(d.name == copyObj.name){ dbo = d; break; }else{ @@ -706,7 +832,7 @@ class DataStandardBatch extends BaseBatch{ } } if(!dbo){ - result = findAddrAliasFromMap(city, region, address); + result = findAddrAliasFromMap(copyObj.city, copyObj.region, copyObj.address); if(result.size()==1){ dbo = result[0]; }else if(result && result.size()>1 && name){ @@ -718,6 +844,11 @@ class DataStandardBatch extends BaseBatch{ } } } + + if(dbo){ + return getFilerMatchDistrict(dbo, copyObj) + } + } return dbo; } @@ -756,7 +887,7 @@ class DataStandardBatch extends BaseBatch{ if(arrays[0] && arrays.size()==2){ def py2 = com.fangjia.core.util.DataUtil.hanyuQuanpin(arrays[0], " ") if(py1 == py2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] } } } @@ -768,11 +899,15 @@ class DataStandardBatch extends BaseBatch{ } - def getOpenApiMatch(def city, def region, def name, def address){ - if(StringUtils.isNotEmpty(name)){ - def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, null, "fuzzy", false) + def getOpenApiMatch(def data){ + if(StringUtils.isNotEmpty(data.name)){ + def jsonObject = OpenApiClient.addressMatch(data.city, data.name, "property", false, data.region, null, "fuzzy", false) if(jsonObject && jsonObject.credit>0 ){ - return [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address, credit:jsonObject.credit] + def dbo = [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address,estate_type:jsonObject.propertyType, credit:jsonObject.credit] + if(dbo){ + return getFilerMatchDistrict(dbo, data) + } + return dbo } } return null; @@ -783,6 +918,9 @@ class DataStandardBatch extends BaseBatch{ def findAllNoMatchData(){ DBObject qdata = new BasicDBObject(); qdata.put("s_date", SHORT_DATE) +// qdata.put("city","北京") + qdata.put("source","安居客") +// qdata.put("name",Pattern.compile("金隅澜湾.*")) qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false)); def ret = [] try{ @@ -948,9 +1086,9 @@ class DataStandardBatch extends BaseBatch{ def arrays = k.split("###") if(arrays[0] && arrays[0]==keyword){ if(arrays.size()>=2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:region, name:arrays[0]] + districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type] } } @@ -962,9 +1100,9 @@ class DataStandardBatch extends BaseBatch{ def arrays = k.split("###") if(arrays[0] && arrays[0]==keyword){ if(arrays.size()>=2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:region, name:arrays[0]] + districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type] } } } @@ -990,18 +1128,18 @@ class DataStandardBatch extends BaseBatch{ def alias = v.alias; if(alias.contains(keyword)){ if(arrays.size()>=2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:region, name:arrays[0]] + districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type] } } }else if(v && v.mapping_alias){ def mappingAlias = v.mapping_alias if(mappingAlias.contains(keyword)){ if(arrays.size()>=2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:region, name:arrays[0]] + districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type] } } } @@ -1016,18 +1154,18 @@ class DataStandardBatch extends BaseBatch{ def alias = v.alias; if(alias.contains(keyword)){ if(arrays.size()>=2){ - districts << [city:city, region:r, name:arrays[0],address:arrays[1]] + districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:r, name:arrays[0]] + districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type] } } }else if(v && v.mapping_alias){ def mappingAlias = v.mapping_alias if(mappingAlias.contains(keyword)){ if(arrays.size()>=2){ - districts << [city:city, region:r, name:arrays[0],address:arrays[1]] + districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:r, name:arrays[0]] + districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type] } } } @@ -1052,9 +1190,9 @@ class DataStandardBatch extends BaseBatch{ def alias = v.addr_alias; if(alias.contains(keyword)){ if(arrays.size()>=2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:region, name:arrays[0]] + districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type] } } } @@ -1068,9 +1206,9 @@ class DataStandardBatch extends BaseBatch{ def alias = v.addr_alias; if(alias.contains(keyword)){ if(arrays.size()>=2){ - districts << [city:city, region:r, name:arrays[0],address:arrays[1]] + districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:r, name:arrays[0]] + districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type] } } } @@ -1093,9 +1231,9 @@ class DataStandardBatch extends BaseBatch{ def arrays = k.split("###") if(arrays && arrays.size()==2 && arrays[1]==keyword){ if(arrays.size()>=2){ - districts << [city:city, region:region, name:arrays[0],address:arrays[1]] + districts << [city:city, region:region, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:region, name:arrays[0]] + districts << [city:city, region:region, name:arrays[0], estate_type:v.estate_type] } } } @@ -1106,9 +1244,9 @@ class DataStandardBatch extends BaseBatch{ def arrays = k.split("###") if(arrays && arrays.size()==2 && arrays[1]==keyword){ if(arrays.size()>=2){ - districts << [city:city, region:r, name:arrays[0],address:arrays[1]] + districts << [city:city, region:r, name:arrays[0],address:arrays[1], estate_type:v.estate_type] }else{ - districts << [city:city, region:r, name:arrays[0]] + districts << [city:city, region:r, name:arrays[0], estate_type:v.estate_type] } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 68943db41..f954743e4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -24,8 +24,8 @@ class SynchDistrictPrice extends BaseBatch{ // "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"] def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"] - // "365淘房":"365淘房","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","吉屋":"吉屋" - def esSources = ["吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产"] + // "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产" + def esSources = ["中原地产":"中原地产"] static main(args) { @@ -57,7 +57,7 @@ class SynchDistrictPrice extends BaseBatch{ def names = new HashSet() def rets = [] ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>='${mdate}' order by m_date", {d-> - def s = d.city +"#"+d.region+"#"+d.name + def s = d.city +"#"+d.region+"#"+d.name+"#"+d.property_type if(!names.contains(s)){ names << s def data = new BasicDBObject() @@ -67,6 +67,7 @@ class SynchDistrictPrice extends BaseBatch{ data.put("address", d.address) data.put("avg_price", d.avg_price) data.put("source", mgSource) + data.put("estate_type",d.property_type) data.put("s_date", SHORT_DATE) data.put("url", d.url) data.put("c_date", new Date()) @@ -144,7 +145,7 @@ class SynchDistrictPrice extends BaseBatch{ println "["+source +"-"+city + "] add:"+ rets.size()+" 原抓取:"+count }else{ for(def r : rets){ - def obj= findIsExists(r.city, r.region, r.name, SHORT_DATE, source) + def obj= findIsExists(r.city, r.region, r.name, r.estate_type, SHORT_DATE, source) if(obj){ updateIndex++ // thirdPartyColl.remove(new BasicDBObject('_id',obj._id)) @@ -152,6 +153,7 @@ class SynchDistrictPrice extends BaseBatch{ def udata = new BasicDBObject() udata.put("avg_price", r.avg_price) + udata.put("estate_type", r.estate_type) udata.put("m_date",new Date()) thirdPartyColl.update(new BasicDBObject('_id',obj._id), new BasicDBObject('$set', udata)) @@ -181,13 +183,16 @@ class SynchDistrictPrice extends BaseBatch{ } - def findIsExists(def city, def region, def name, def sDate, def source){ + def findIsExists(def city, def region, def name, estate_type, def sDate, def source){ def q = new BasicDBObject() q.put("source", source) q.put("s_date", sDate) q.put("city", city) q.put("region", region) q.put("name", name) + if(estate_type){ + q.put("estate_type",estate_type) + } def obj = thirdPartyColl.findOne(q) return obj } From 1d561a3f55a8a103c79944c818145ae38cfff59d Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 19 Oct 2022 17:40:02 +0800 Subject: [PATCH 2/6] update --- .../script/ccbevaluation/ExportOracleData.groovy | 16 ++++------------ .../script/ccbevaluation/TradeStatLog.groovy | 14 +++++++------- .../com/fangjia/script/gj/YouDaDataParser.groovy | 4 ++-- .../thirdPriceCrawl/DataStandardBatch.groovy | 6 ++++-- 4 files changed, 17 insertions(+), 23 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 0fbc2daa8..e1e17ee47 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -16,13 +16,15 @@ class ExportOracleData { "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") def cityCodeMap = CcbCommonUtils.findCityCodeValue() + def citys = null static main(args) { def task = new ExportOracleData() - task.run() + task.run(args) } - def run() { + def run(args) { + citys = CcbCommonUtils.getRunningCitys(args) // def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") @@ -131,7 +133,6 @@ class ExportOracleData { def exportBigdataPoiData() { - def citys = CcbCommonUtils.citys_all def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) @@ -200,7 +201,6 @@ class ExportOracleData { // def citys = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"] // def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def allRows = [] - def citys = CcbCommonUtils.citys_all for(def city : citys) { def cityCode = cityCodeMap.get(city) def sql = "select CITYID, COMMID, BUILDTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, WORKDATE from bigdata_commbuildtype_gj where CITYID='${cityCode}'"; @@ -233,7 +233,6 @@ class ExportOracleData { } def exportBigdataDistrictData() { - def citys = CcbCommonUtils.citys_all def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) @@ -275,7 +274,6 @@ class ExportOracleData { def exportGuidedPrice() { - def citys = CcbCommonUtils.citys_all def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) @@ -314,7 +312,6 @@ class ExportOracleData { } def exportCommunityGuidedPrice(def date) { - def citys = CcbCommonUtils.citys_all def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) @@ -354,7 +351,6 @@ class ExportOracleData { def exportBigdataDeal(def dealDate, def new_dists) { def allRows = [] - def citys = CcbCommonUtils.citys_all for(def city : citys) { def cityCode = cityCodeMap.get(city) if(new_dists){ @@ -530,7 +526,6 @@ class ExportOracleData { } def headers = [] def index = 0 - def citys = CcbCommonUtils.citys_all for(def city : citys) { def cityCode = cityCodeMap.get(city) def allRows = [] @@ -588,7 +583,6 @@ class ExportOracleData { def exportBigdataCommunity(){ - def citys = CcbCommonUtils.citys_all def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) @@ -648,8 +642,6 @@ class ExportOracleData { def exportBigdataForsale(publishDate, new_dists){ //["北京","上海"] - def citys = CcbCommonUtils.citys_all; - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index b2bbe9713..c2f488b5f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -73,15 +73,15 @@ class TradeStatLog { } def run(){ - def citys = CcbCommonUtils.citys; + def citys = CcbCommonUtils.citys_92; def rets = [] - rets << ["城市","6月总量","7月总量","8月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] - def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"] + rets << ["城市","7月总量","8月总量","9月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] + def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"] //def startDate = "2021-05-01" - def startDate = "2022-06-01" - def date = "2022-07-01" - def endDate = "2022-08-01" - def newDate = "2022-09-01" + def startDate = "2022-07-01" + def date = "2022-08-01" + def endDate = "2022-09-01" + def newDate = "2022-10-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index 616012302..573083bfb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -48,13 +48,13 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2022-08-01" + static def statDate = "2022-09-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo // task.esfParser("D:\\数据\\友达\\2022年9月\\查询1.DBF");//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es - task.esfhouseToSaveEs();//二手房保存到es +// task.esfhouseToSaveEs();//二手房保存到es // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index 721bf7025..747db86c1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -681,7 +681,7 @@ class DataStandardBatch extends BaseBatch{ }else{ def priceMap = findHousePrice(dbo.city, dbo.region, dbo.name, DateUtils.format(DateUtil.add(new Date(), -180),"yyyyMMdd").toInteger()) def price = priceMap.get(dbo.estate_type) - if(price){ + if(price && data.avg_price && data.avg_price.toString().isNumber()){ def pre = (data.avg_price-price)*100/price if(pre>=-20 && pre<=20){ return dbo; @@ -713,6 +713,8 @@ class DataStandardBatch extends BaseBatch{ key = "商住" }else if( data.estate_type.contains("别墅")){ key = "别墅" + }else{ + key = "其它" } def houses = null if(key){ @@ -919,7 +921,7 @@ class DataStandardBatch extends BaseBatch{ DBObject qdata = new BasicDBObject(); qdata.put("s_date", SHORT_DATE) // qdata.put("city","北京") - qdata.put("source","安居客") +// qdata.put("source","安居客") // qdata.put("name",Pattern.compile("金隅澜湾.*")) qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false)); def ret = [] From f61e77948cb4d3d2b6508bb70c38210df0ca1f15 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 19 Oct 2022 17:46:16 +0800 Subject: [PATCH 3/6] update bug --- .../fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index 747db86c1..91d3dee7e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -483,7 +483,7 @@ class DataStandardBatch extends BaseBatch{ if(district){ type = 10; }else if(districts.size()==1){ - district = getFilerMatchDistrict(districts[0], data.estate_type, avg_price); + district = getFilerMatchDistrict(districts[0], data); if(district){ type = 14 } @@ -562,7 +562,7 @@ class DataStandardBatch extends BaseBatch{ } if(district){ - return getFilerMatchDistrict(district, data.estate_type, data.avg_price) + return getFilerMatchDistrict(district, data) } return district; } From 54ff9f60bd41eef31a9afdae645c3186792d1d4d Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 20 Oct 2022 10:38:52 +0800 Subject: [PATCH 4/6] =?UTF-8?q?=E7=AC=AC=E4=B8=89=E6=96=B9=E5=9D=87?= =?UTF-8?q?=E4=BB=B7=E6=9C=88=E4=BB=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/ThirdPartyPriceBatch.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index c4a27815e..7bcb09230 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202208//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202206 - def endDate = 202208 + def sDate= 202209//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202207 + def endDate = 202209 def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") From 9814ae21fe9f48a6b539b552bd7bab804709fc4e Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 20 Oct 2022 11:26:56 +0800 Subject: [PATCH 5/6] =?UTF-8?q?=E5=9F=8E=E5=B8=82=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/BigdataDealBatch.groovy | 72 +++++++++---------- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 2 +- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 89b9f7f35..6d98b0d92 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -63,44 +63,44 @@ class BigdataDealBatch extends BaseBatch{ curDate = args[1].toString().toInteger() } -// for(def city : citys){ -//// if(!ccbCitys.contains(city)){ -// addDealPriceMatch(city); -//// } -// } - def rows = [] - def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] - def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ - def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def ppeDists = [:] - - def statMap = [:]//getDistrictStatByCity(city, weekend) - if(city=="上海"){ - def youDataTrade = getYouDaData(statMap) - def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") - rows.addAll(youdaDatas) - println city +" "+youdaDatas.size() - } - - - def trades = getCityData(city, statMap, ppeDists) - def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") - rows.addAll(rets) - println city +" 成交共:"+rets.size() - - def cityDatas = [] - cityDatas << headers - cityDatas.addAll(rows) +// if(!ccbCitys.contains(city)){ + addDealPriceMatch(city); +// } } - - def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" - if(CommonDatautils.isWindow()){ - fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" - } - - FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", true); - println "txt 生成 " + rows.size() +// def rows = [] +// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] +// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +// for(def city : citys){ +// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) +// def ppeDists = [:] +// +// def statMap = [:]//getDistrictStatByCity(city, weekend) +// if(city=="上海"){ +// def youDataTrade = getYouDaData(statMap) +// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") +// rows.addAll(youdaDatas) +// println city +" "+youdaDatas.size() +// } +// +// +// def trades = getCityData(city, statMap, ppeDists) +// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") +// rows.addAll(rets) +// println city +" 成交共:"+rets.size() +// +// def cityDatas = [] +// cityDatas << headers +// cityDatas.addAll(rows) +// } +// +// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" +// if(CommonDatautils.isWindow()){ +// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" +// } +// +// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", true); +// println "txt 生成 " + rows.size() } def createData(def rets, cityCodeDatas, distCodeMap, source){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 7bcb09230..f0d9b5eb2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -42,7 +42,7 @@ class ThirdPartyPriceBatch extends BaseBatch { @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + def citys = CcbCommonUtils.getRunningCitys(args) def rows = [] def index = 0 def weekend = DateUtils.getFirstDayOfWeek(new Date()) From 1be1d3b444775495c92ab085fe03a5b54f999a50 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 20 Oct 2022 13:56:57 +0800 Subject: [PATCH 6/6] =?UTF-8?q?=E6=88=90=E4=BA=A4=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/BigdataDealBatch.groovy | 72 +++++++++---------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 6d98b0d92..89b9f7f35 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -63,44 +63,44 @@ class BigdataDealBatch extends BaseBatch{ curDate = args[1].toString().toInteger() } - for(def city : citys){ -// if(!ccbCitys.contains(city)){ - addDealPriceMatch(city); -// } - } -// def rows = [] -// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] -// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() // for(def city : citys){ -// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) -// def ppeDists = [:] -// -// def statMap = [:]//getDistrictStatByCity(city, weekend) -// if(city=="上海"){ -// def youDataTrade = getYouDaData(statMap) -// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") -// rows.addAll(youdaDatas) -// println city +" "+youdaDatas.size() -// } -// -// -// def trades = getCityData(city, statMap, ppeDists) -// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") -// rows.addAll(rets) -// println city +" 成交共:"+rets.size() -// -// def cityDatas = [] -// cityDatas << headers -// cityDatas.addAll(rows) +//// if(!ccbCitys.contains(city)){ +// addDealPriceMatch(city); +//// } // } -// -// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" -// if(CommonDatautils.isWindow()){ -// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" -// } -// -// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", true); -// println "txt 生成 " + rows.size() + def rows = [] + def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + for(def city : citys){ + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + def ppeDists = [:] + + def statMap = [:]//getDistrictStatByCity(city, weekend) + if(city=="上海"){ + def youDataTrade = getYouDaData(statMap) + def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") + rows.addAll(youdaDatas) + println city +" "+youdaDatas.size() + } + + + def trades = getCityData(city, statMap, ppeDists) + def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") + rows.addAll(rets) + println city +" 成交共:"+rets.size() + + def cityDatas = [] + cityDatas << headers + cityDatas.addAll(rows) + } + + def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" + if(CommonDatautils.isWindow()){ + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" + } + + FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", true); + println "txt 生成 " + rows.size() } def createData(def rets, cityCodeDatas, distCodeMap, source){