diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 75b1a0e45..42f56e208 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -82,9 +82,10 @@ class CcbCommonUtils { return rets } - public static def findGuidePrice(city){ + public static def findGuidePrice(city, startDate, endDate){ def q = new BasicDBObject() q.put("city", city) + q.put('c_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt', DateUtils.parseFromTightYYYYMMDD(endDate))); def dbcursor = DISTRICT_GUIDE_PRICE_COLL.find(q) def rets = [] for(def d : dbcursor){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy index b64fdcd1a..2f73eda31 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy @@ -25,7 +25,7 @@ class DistrictGuidePriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findGuidePrice(city) + def prices = CcbCommonUtils.findGuidePrice(city, "2022-01-1", "2022-02-1") if(prices){ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 24dbf7876..736221526 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -24,12 +24,14 @@ class ExportOracleData { } def run() { -// + def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); + + def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); // exportGuidedPrice() -// exportCommunityGuidedPrice() + exportCommunityGuidedPrice(lastMonth.toString()+"28") // // //exportThirdPartyPrice()//废弃掉,老表 -// exportThirdPartyPrice2(202111) +// exportThirdPartyPrice2(lastMonth) // // // @@ -40,20 +42,20 @@ class ExportOracleData { // exportCommunityStaging() // exportCommunityBuileType() // exportBigdataDistrictData() - exportBigdataCommunity() +// exportBigdataCommunity() // exportBigdataDeal("2021-11-01") -// exportAvmEvaluationGJNB(202112) -// exportAVMEvaluationItemGJNB(202112) -// exportCommunityBasePrice(202112) +// exportAvmEvaluationGJNB(curMonth) +// exportAVMEvaluationItemGJNB(curMonth) +// exportCommunityBasePrice(curMonth) // -// exportCityPrice(202112) -// exportRegionPrice(202112) +// exportCityPrice(curMonth) +// exportRegionPrice(curMonth) -// exportCommunityBasePriceGj(202112) +// exportCommunityBasePriceGj(curMonth) -// exportCommunityPdescribeGj(202112) -// exportCommCaseDetailGj(202112) +// exportCommunityPdescribeGj(curMonth) +// exportCommCaseDetailGj(curMonth) // exportLastFile() } @@ -255,26 +257,26 @@ class ExportOracleData { println "GUIDED_PRICE_GJ 上传Ftp完成......" } - def exportCommunityGuidedPrice() { - def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj"; + def exportCommunityGuidedPrice(def date) { + def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where createtime>=to_date('${date}','yyyyMMdd')"; def allRows = [] -// sqlInstance.eachRow(sql, { r -> -// def data = [] -// data << r.ID -// data << r.COMMID -// data << r.COMMNAME -// data << r.CITYID -// data << r.CITYNAME -// data << r.DISTID -// data << r.DISTNAME -// data << r.AVGPRICE -// data << r.SOURCETYPE -// data << r.REMARK -// data << r.CREATETIME -// data << r.UPDATETIME -// data << r.SOURCECODE -// allRows << data -// }); + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMID + data << r.COMMNAME + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.AVGPRICE + data << r.SOURCETYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 03893ad8b..ef1f1d1ef 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2021-10-1", "2021-11-1") + def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-1", "2022-02-1") if(prices){ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy index c6fa7f82c..84f2232df 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy @@ -4,6 +4,8 @@ import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import com.fangjia.core.RegionData +import com.fangjia.script.gj.district.SeaweedDataMatchProcess class LoadGuide { public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017) @@ -14,13 +16,31 @@ class LoadGuide { } def run(){ +// match("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx") + +// saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞指导价218.xlsx") // saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞二手房政府指导价.xls") // saveChengduoPrice("D:\\数据\\建行\\房产估值\\指导价\\成都部分住宅小区二手房成交参考价格201-286-258.xlsx") // savePrice("D:\\数据\\建行\\房产估值\\指导价\\绍兴指导价小区_0909.xlsx") // savePrice("D:\\数据\\建行\\房产估值\\指导价\\宁波交易参考价格.xlsx") -// savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安成交参考价格.xlsx") + savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx") // savePrice("D:\\数据\\建行\\房产估值\\指导价\\无锡100 广州96 二手住房成交参考价格表.xlsx") - savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx") +// savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx") + + } + def match(def filename){ + def city = "西安" + ExcelUtils.modify(filename,{d-> + def newRegion = RegionData.formatRegion(city, d['A']) + if(newRegion){ + def rMap = SeaweedDataMatchProcess.doMatch(city, newRegion, d['B'], null); + if(rMap){ + d['D'] = city + d['E'] = rMap.region + d['F'] = rMap.name + } + } + },0) } @@ -32,8 +52,8 @@ class LoadGuide { def o = new BasicDBObject() o.city = d['E'] o.region = d['A'] - o.name = d['C'] - o.avg_price = d['D'].toString().toDouble().toInteger() + o.name = d['B'] + o.avg_price = d['C'].toString().toDouble().toInteger() o.match_city = d['E'] o.match_region = d['F'] o.match_name = d['G'] @@ -57,11 +77,11 @@ class LoadGuide { o.city = d['E'] o.region = d['A'] o.name = d['B'] - o.avg_price = d['D'].toString().toDouble().toInteger() + o.avg_price = d['C'].toString().toDouble().toInteger() o.match_city = d['E'] o.match_region = d['F'] o.match_name = d['G'] - o.house_count = d['C'].toString().toDouble().toInteger() + // o.house_count = d['C'].toString().toDouble().toInteger() o.source_type = 1 //1:成交参考价 2:参考价 o.c_date = new Date() rets << o diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy index 8373b928c..8ec8ba66a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy @@ -4,7 +4,107 @@ import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.district.SeaweedDataMatchProcess import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + + +DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43") + +//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d -> +// def newRegion = RegionData.formatRegion(d['A'], d['B']) +// if(newRegion && !d['K']) { +// def rMap = SeaweedDataMatchProcess.doMatch(d['A'], newRegion, d['C'], d['D']+d['E']); +// if(!rMap && d['G']){ +// rMap = SeaweedDataMatchProcess.doMatch(d['A'], newRegion, d['G'], d['H']); +// } +// if(rMap){ +// d['O'] = rMap.city +// d['P'] = rMap.region +// d['Q'] = rMap.name +// d['R'] = rMap.address +// } +// } +//}); + + +//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\高德反查数据3_2003.xlsx",{d-> +// def newRegion = RegionData.formatRegion(d['A'], d['B']) +// if(newRegion && !d['K']){ +// +// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'],"district") +// if(dist){ +// d['K']="完全一致" +// d['L'] = dist.estate_type2 +// d['M'] = dist.name_cat +// }else{ +// dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['C'],"district") +// if(dist){ +// d['K']="映射别名" +// d['L'] = dist.estate_type2 +// d['M'] = dist.name_cat +// }else{ +// def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['C'],"district") +// if(rets){ +// dist = rets[0] +// } +// if(rets){ +// d['K']="别名" +// d['L'] = dist.estate_type2 +// d['M'] = dist.name_cat +// } +// } +// } +// } +// +//},0) + +//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\高德反查数据3_2003.xlsx",{d-> +// def newRegion = RegionData.formatRegion(d['A'], d['B']) +// if(newRegion && !d['K'] && d['G']){ +// +// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['G'],"district") +// if(dist){ +// d['K']="完全一致(高德)" +// d['L'] = dist.estate_type2 +// d['M'] = dist.name_cat +// }else{ +// dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['G'],"district") +// if(dist){ +// d['K']="映射别名(高德)" +// d['L'] = dist.estate_type2 +// d['M'] = dist.name_cat +// }else{ +// def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['G'],"district") +// if(rets){ +// dist = rets[0] +// } +// if(rets){ +// d['K']="别名(高德)" +// d['L'] = dist.estate_type2 +// d['M'] = dist.name_cat +// } +// } +// } +// } +// +//},0) + + +//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\价格过滤小区融合情况_1209.xlsx",{d-> +// def dist = CommonDatautils.findDistrictFieldById(d['F'], ["address","lat","lng"]) +// if(dist){ +// d['Z'] = dist.address +// if(dist.lat){ +// d['AA'] = dist.lat/10000000000.0 +// d['AB'] = dist.lng/10000000000.0 +// } +// } +//},0) //def cityDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\小区\\重庆高德反查数据_1529.xlsx",0,1) //def cityDataMap = [:] @@ -74,54 +174,156 @@ def city = " //},0) -//ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例.xlsx",{d-> +//ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例(合并).xlsx",{d-> // def obj = CommonDatautils.findDistrict("广州", d['C'], d['D'], "district") // if(obj){ -// def year = CommonUtils.getDistrictAge(obj) -// if(year){ -// d['H'] = year -// } -// if(obj.alias){ -// d['K'] = obj.alias.join(" ") +//// def year = CommonUtils.getDistrictAge(obj) +//// if(year){ +//// d['H'] = year +//// } +//// if(obj.alias){ +//// d['K'] = obj.alias.join(" ") +//// } +// if(obj.mapping_alias){ +// d['X'] = obj.mapping_alias.join(" ") // } // // } //},0) -def data1s = ExcelUtils.read("D:\\数据\\广发\\图片转换汇总1.8.xlsx",0, 1) -def scoreMap = [:] -for(def d : data1s){ - scoreMap.put(d['B'], d) -} +//def data1s = ExcelUtils.read("D:\\数据\\广发\\图片转换汇总1.8.xlsx",0, 1) +//def scoreMap = [:] +//for(def d : data1s){ +// scoreMap.put(d['B'], d) +//} +// +//def data2s = ExcelUtils.read("D:\\数据\\广发\\图片转换汇总1.8.xlsx",1, 1) +//def levelMap = [:] +//for(def d : data2s){ +// levelMap.put(d['E'], d) +//} +// +//ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例.xlsx",{d-> +// if(d['E']){ +// def row = scoreMap.get(d['E']) +// if(row){ +// d['I'] = row['G'] +// d['J'] = row['H'] +// d['K'] = row['I'] +// d['L'] = row['J'] +// d['M'] = row['K'] +// d['N'] = row['L'] +// d['O'] = row['M'] +// } +// +// def data = levelMap.get(d['E']) +// if(data){ +// d['P'] = data['G'] +// d['Q'] = data['H'] +// d['R'] = data['I'] +// d['S'] = data['J'] +// d['T'] = data['K'] +// d['U'] = data['L'] +// d['V'] = data['M'] +// } +// } +//},0) -def data2s = ExcelUtils.read("D:\\数据\\广发\\图片转换汇总1.8.xlsx",1, 1) -def levelMap = [:] -for(def d : data2s){ - levelMap.put(d['E'], d) -} +//ExcelUtils.modify("D:\\数据\\成交\\福州成交.xlsx",{d-> +// def o = new BasicDBObject() +// o.area = d['D'].toDouble() +// o.avg_price = d['F'].toInteger() +// o.city = "福州" +// o.district_name = d['C'] +// o.region = d['B'] +// o.source = "贝壳找房" +// o.trade_date = DateUtils.parseFromTightYYYYMMDD(d['M']) +// o.total_price = d['E'].toInteger()*10000 +// if(d['I']){ +// o.hall = d['I'].toInteger() +// } +// +// o.m_date = new Date() +// if(d['H']){ +// o.room = d['H'].toInteger() +// } +// +// o.create_date = new Date() +// o.status = 2 +// DEAL_PRICE.save(o) +// +//},0) -ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例.xlsx",{d-> - if(d['E']){ - def row = scoreMap.get(d['E']) - if(row){ - d['I'] = row['G'] - d['J'] = row['H'] - d['K'] = row['I'] - d['L'] = row['J'] - d['M'] = row['K'] - d['N'] = row['L'] - d['O'] = row['M'] - } - def data = levelMap.get(d['E']) - if(data){ - d['P'] = data['G'] - d['Q'] = data['H'] - d['R'] = data['I'] - d['S'] = data['J'] - d['T'] = data['K'] - d['U'] = data['L'] - d['V'] = data['M'] +//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d -> +// if(d['K'] && d['K']=="别名"){ +// def newRegion = RegionData.formatRegion(d['A'], d['B']) +// if(newRegion){ +// def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['C'],"district") +// def dist = null +// if(rets){ +// dist = rets[0] +// } +// if(dist){ +// def alias = dist.mapping_alias +// if(!alias){ +// alias = [] +// } +// if(!alias.contains(d['C'])){ +// alias << d['C'] +// } +// def udata = new BasicDBObject() +// udata.put("mapping_alias",alias) +// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) +// } +// } +// } +//}); + + +//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d -> +// if(d['K'] && d['K']=="映射别名(高德)" && d['G']){ +// def newRegion = RegionData.formatRegion(d['A'], d['B']) +// if(newRegion){ +// dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['G'],"district") +// if(dist){ +// def alias = dist.mapping_alias +// if(!alias){ +// alias = [] +// } +// if(!alias.contains(d['C'])){ +// alias << d['C'] +// } +// def udata = new BasicDBObject() +// udata.put("mapping_alias",alias) +// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) +// } +// } +// } +//}); + + +ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d -> + if(d['K'] && d['K']=="别名(高德)" && d['G']){ + def newRegion = RegionData.formatRegion(d['A'], d['B']) + if(newRegion){ + def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['G'],"district") + def dist = null + if(rets){ + dist = rets[0] + } + if(dist){ + def alias = dist.mapping_alias + if(!alias){ + alias = [] + } + if(!alias.contains(d['C'])){ + alias << d['C'] + } + def udata = new BasicDBObject() + udata.put("mapping_alias",alias) + DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) + } } } -},0) +}); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index c7847b806..683312af1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -27,6 +27,7 @@ class ExportDistrictRongHe extends BaseBatch{ def citys = CcbCommonUtils.citys; def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + //检查ok文件是否上传 // exportAllBigdataDealFJW() for(def city : citys) { def cityCode = cityCodeDataMap.get(city) @@ -47,6 +48,7 @@ class ExportDistrictRongHe extends BaseBatch{ exportBigdataCommlayoutpicFjw(cityCode) } exportEvaluateRate() + exportOk() } @@ -593,4 +595,14 @@ class ExportDistrictRongHe extends BaseBatch{ println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......" } + def exportOk(){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok" + if(CommonDatautils.isWindow()){ + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok" + } + FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); + CcbFtpUtils.ftpUpload(dataCodeFileName, curDate) + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/DownPdfCheck.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/DownPdfCheck.groovy index bd195836d..2e5201a8c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/DownPdfCheck.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/DownPdfCheck.groovy @@ -13,7 +13,34 @@ class DownPdfCheck { static main(args) { def task = new DownPdfCheck(); - task.run(); +// task.run(); + + task.czLogRun() + } + + def czLogRun(){ + def files = getAllFiles("D:\\数据\\常州") + if(files){ + for(def filepath : files){ + read(filepath); + } + } + + } + + + def read(file){ + // File file = new File(fileName); + def index = 0 + + file.eachLine{it-> + if(it.contains("4ead0347b4055be6989e726e")||it.contains("61a2e0")){ + index++ + + println it + } + } + } def run(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy index 87a2d9962..e8cd3476a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy @@ -8,6 +8,7 @@ import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.jusfoun.JusfounCommon import com.fangjia.script.util.DateUtils import org.apache.commons.codec.digest.DigestUtils +import org.apache.commons.io.FileUtils import java.util.concurrent.TimeUnit @@ -79,6 +80,14 @@ class DistrictPriceExportBatch extends BaseBatch{ def lastDate = new Date() def startDate = DateUtils.getFirstDayOfMonth(lastDate, -1) +// List list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202111.csv")) +// def map2 = [:] +// for(def s : list2) { +// def row = s.split("@\\|@") +// def key = row[2] + "##" + row[3] + "##" + row[4] +// map2.put(key, row[5]) +// } + def result = [] for(def rMap : districts){ def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate) @@ -90,6 +99,25 @@ class DistrictPriceExportBatch extends BaseBatch{ m << rMap.region m << rMap.name m << dbo.get("avg_price") +// def lastMonthPrice = map2.get(rMap.city + "##" + rMap.region+"##"+rMap.name) +// if(lastMonthPrice && dbo.get("avg_price")){ +// lastMonthPrice = lastMonthPrice.toInteger() +// def markup = (dbo.get("avg_price")-lastMonthPrice)*100.0/lastMonthPrice +// if(Math.abs(markup)>10){ +// def random = 0 +// if(dbo.get("avg_price")-lastMonthPrice>=0){ +// random = com.fangjia.core.util.RandomUtil.random(1, 9) +// }else{ +// random = com.fangjia.core.util.RandomUtil.random(-9, 0) +// } +// def newPrice = lastMonthPrice*random/100 + lastMonthPrice +// m << newPrice.toInteger() +// }else{ +// m << dbo.get("avg_price") +// } +// }else{ +// m << dbo.get("avg_price") +// } m << curDate.toInteger(); result << m } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy index 88100888a..0231b21c0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy @@ -48,51 +48,51 @@ import org.bson.types.ObjectId // return location; //} -//List list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202109.csv")) -//def map1 = [:] -//for(def s : list1){ -// def row = s.split("@\\|@") -// def key = row[2]+"##"+row[3]+"##"+row[4] -// map1.put(key, row[5]) -//} -// -//List list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202108.csv")) -//def map2 = [:] -//for(def s : list2) { -// def row = s.split("@\\|@") -// def key = row[2] + "##" + row[3] + "##" + row[4] -// map2.put(key, row[5]) -//} -//def rows = [] -//map1.each {k,v-> -// def row = [] -// def keys = k.split("##") -// def lastPrice = map2.get(k) -// rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString()] -//} -//ExcelUtils.write("D:\\数据\\福州\\价格比对.xlsx", rows, "数据") -ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州涨幅在10%以上小区.xlsx",{d-> - def obj = CommonDatautils.queryDistrictPrice(d['A'],d['B'], d['C'], DateUtils.parseFromTightByPattern("2021-10-10","yyyy-MM-dd")) - if(obj){ - if(obj.value_type == 1){ - d["G"] = "指导价" - }else if(obj.value_type == 2){ - d["G"] ="自身样本" - }else if(obj.value_type == 3){ - d["G"] = "第三方" - }else if(obj.value_type == 4){ - d["G"] = "交易数据" - }else if(obj.value_type == 5){ - d["G"] ="周边楼盘" - }else if(obj.value_type == 6){ - d["G"] ="推导" - }else if(obj.value_type == 9){ - d["G"] = "神经网络" - }else if(obj.value_type == 10){ - d["G"] = "无方式" - } - } -},0) +List list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202112.csv")) +def map1 = [:] +for(def s : list1){ + def row = s.split("@\\|@") + def key = row[2]+"##"+row[3]+"##"+row[4] + map1.put(key, row[5]) +} + +List list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202111.csv")) +def map2 = [:] +for(def s : list2) { + def row = s.split("@\\|@") + def key = row[2] + "##" + row[3] + "##" + row[4] + map2.put(key, row[5]) +} +def rows = [] +map1.each {k,v-> + def row = [] + def keys = k.split("##") + def lastPrice = map2.get(k) + rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString()] +} +ExcelUtils.write("D:\\数据\\福州\\价格比对12-2.xlsx", rows, "数据") +//ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州涨幅在10%以上小区.xlsx",{d-> +// def obj = CommonDatautils.queryDistrictPrice(d['A'],d['B'], d['C'], DateUtils.parseFromTightByPattern("2021-10-10","yyyy-MM-dd")) +// if(obj){ +// if(obj.value_type == 1){ +// d["G"] = "指导价" +// }else if(obj.value_type == 2){ +// d["G"] ="自身样本" +// }else if(obj.value_type == 3){ +// d["G"] = "第三方" +// }else if(obj.value_type == 4){ +// d["G"] = "交易数据" +// }else if(obj.value_type == 5){ +// d["G"] ="周边楼盘" +// }else if(obj.value_type == 6){ +// d["G"] ="推导" +// }else if(obj.value_type == 9){ +// d["G"] = "神经网络" +// }else if(obj.value_type == 10){ +// d["G"] = "无方式" +// } +// } +//},0) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index 9968280b4..2f94c98c9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202111.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202112.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy index a53edd4d5..f2fbed25d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -62,7 +62,8 @@ class DistrictIndexBatch extends BaseBatch{ def run(){ def cityCodeDatas = CcbCommonUtils.findCityCodeValue() - def citys = ["上海"]//CommonDatautils.findAllCity(); + def citys = CommonDatautils.findAllCity(); + citys.remove("上海") for(def city : citys){ def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2"]) // dists = dists[0..20] @@ -156,13 +157,19 @@ class DistrictIndexBatch extends BaseBatch{ fileName = "D:\\数据\\评级\\上海综合潜力.dat" } - FileUtils.batchCreateFile(datas, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile(datas, fileName, headers, "utf-8", true); // ExcelUtils.write("D:\\数据\\评级\\上海综合潜力.xlsx", datas, "数据") } def getHousePrice(d, dists, priceMap, citycode){ //def areas = findHouse(d, 20200101, 20220101) - def areas = findHouseByOracle(d._id.toString(), citycode); + + def areas = [] + if(citycode){ + areas = findHouseByOracle(d._id.toString(), citycode); + }else{ + areas = findHouse(d, 20200101, 20220101) + } def avg_area = null if(areas){ avg_area = areas.sum()/areas.size(); @@ -188,24 +195,27 @@ class DistrictIndexBatch extends BaseBatch{ if(selectDists){ println d.region+" "+d.name+" 周边同建筑年代楼盘:"+selectDists.size() -// def allAreas = [] + def allAreas = [] def ids = [] - for(def sd : selectDists){ -// //def areaDatas = findHouse(sd, 20190101, 20220101 -// def areaDatas = findHouseByOracle(sd._id.toString()); -// allAreas.addAll(areaDatas) - ids << sd._id.toString() - } - if(ids.size()>=901){ - ids = ids[0..900] - } + if(citycode){ + for(def sd : selectDists){ + ids << sd._id.toString() + } + if(ids.size()>=901){ + ids = ids[0..900] + } - def idstr = ids.join("','") - def allAreas = findHouseByOracle2(idstr,citycode); + def idstr = ids.join("','") + allAreas = findHouseByOracle2(idstr,citycode); + }else{ + for(def sd : selectDists){ + def areaDatas = findHouse(sd, 20190101, 20220101) + allAreas.addAll(areaDatas) + } + } if(allAreas){ avg_area = allAreas.sum()/allAreas.size(); } - } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/CrawalUtil.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/CrawalUtil.groovy index a284b5c6f..d283bbf5c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/CrawalUtil.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/CrawalUtil.groovy @@ -98,9 +98,9 @@ class CrawalUtil { } - static def getDocumentByABuYun(def url, def errorMsg, def charset, def language = null) { + static def getDocumentByABuYun(def url, def errorMsg, def charset, def language = null, def isJson=false) { def flag = true; - Document document = null; + def document = null; def count = 0; while (flag) { def content = null; @@ -121,7 +121,11 @@ class CrawalUtil { } } if (content) { - document = Jsoup.parse(content); + if(!isJson){ + document = Jsoup.parse(content); + }else{ + document = content + } } if (!content || (content && errorMsg && content.contains(errorMsg))) { count++; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index 6dedf2c7b..1d80f9b2a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -364,7 +364,7 @@ class DataStandardBatch extends BaseBatch{ def standardDistrict(def data, def avg_price){ //名称匹配,名称别名匹配,地址别名匹配 //def district = getMatchName(data.city, data.region, data.name, data.address,"district") - data.name = data.name.replace("(","").replace(")","") + data.name = data.name.replace("(","").replace(")","").replace("数据由万科物业提供","").trim() def district = null; def type = 0; if(data.name.contains("写字楼")||data.name.contains("办公楼")||data.name.contains("商业")){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index 6566d5a16..5d972bd90 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -40,11 +40,11 @@ class DataStat { def run(){ - def source = "城市房产"// - def citys = findCrawlCity(source, 202110); + def source = "58同城"// + def citys = findCrawlCity(source, 202111); for(def city : citys){ - def count1 = findCount(source, city, 202110); - def count2 = findCount(source, city, 202111); + def count1 = findCount(source, city, 202111); + def count2 = findCount(source, city, 202112); println city +" "+count1+" "+count2 // println count1 // println "--- ----" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index ae2639418..10c39958e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -12,6 +12,7 @@ import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject +import net.sf.json.JSONObject import org.jsoup.nodes.Element import org.jsoup.select.Elements @@ -35,17 +36,16 @@ class Esf58Crawl extends BaseBatch{ def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def static final distMap = [:] - def static useDistKeys = [] + def static useDistKeys = ["a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z","不","了","这","中","大","有","和","人","是","在","一","的","为","上","个","国","我","以","要","他","时","来","用","们","生","到","作","地","于","出","就","分","对","成","会","可","主","发","年","动","同","工","也","能","下","过","子","说","产","种","面","而","方","后","多","定","行","学","法","所","民","得","经","十","三","之","进","着","等","部","度","家","电","力","里","如","水","化","高","自","二","理","起","小","物","现","实","加","量","都","两","体","制","机","当","使","点","从","业","本","去","把","性","好","应","开","它","合","还","因","由","其","些","然","前","外","天","政","四","日","那","社","义","事","平","形","相","全","表","间","样","与","关","各","重","新","线","内","数","正","心","反","你","明","看","原","又","么","利","比","或","但","质","气","第","向","道","命","此","变","条","只","没","结","解","问","意","建","月","公","无","系","军","很","情","者","最","立","代","想","已","通","并","提","直","题","党","程","展","五","果","料","象","员","革","位","入","常","文","总","次","品","式","活","设","及","管","特","件","长","求","老","头","基","资","边","流","路","级","少","图","山","统","接","知","较","长","将","组","见","计","别","她","手","角","期","根","论","运","农","指","几","九","区","强","放","决","西","被","干","做","必","战","先","回","则","任","取","据","处","队","南","给","色","光","门","即","保","治","北","造","百","规","热","领","七","海","地","口","东","导","器","压","志","世","金","增","争","济","阶","油","思","术","极","交","受","联","什","认","六共","权","收","证","改","清","已","美","再","采","转","更","单","风","切","打","白","教","速","花","带","安","场","身","车","例","真","务","具","万","每","目","至","达","走","积","示","议","声","报","斗","完","类","八","离","华","名","确","才","科","张","信","马","节","话","米","整","空","元","况","今","集","温","传","土","许","步","群","广","石","记","需","段","研","界","拉","林","律","叫","且","究","观","越","织","装","影","算","低","持","音","众","书","布","复","容","儿","须","际","商","非","验","连","断","深","难","近","矿","千","周","委","素","技","备","半","办","青","省","列","习","响","约","支","般","史","感","劳","便","团","往","酸","历","市","克","何","除","消","构","府","称","太","准","精","值","号","率","族","维","划","选","标","写","存","候","毛","亲","快","效","斯","院","查","江","型","眼","王","按","格","养","易","置","派","层","片","始","却","专","状","育","厂","京","识","适","属","圆","包","火","住","调","满","县","局","照","参","红","细","引","听","该","铁","价","严","首","底","液","官","德","调","随","病","苏","失","尔","死","讲","配","女","黄","推","显","谈","罪","神","艺","呢","席","含","企","望","密","批","营","项","防","举","球","英","氧","势","告","李","台","落","木","帮","轮","破","亚","师","围","注","远","字","材","排","供","河","态","封","另","施","减","树","溶","怎","止","案","言","士","均","武","固","叶","鱼","波","视","仅","费","紧","爱","左","章","早","朝","害","续","轻","服","试","食","充","兵","源","判","护","司","足","某","练","差","致","板","田","降","黑","犯","负","击","范","继","兴","似","余","坚","曲","输","修","的","故","城","夫","够","送","笔","船","占","右","财","吃","富","春","职","觉","汉","画","功","巴","跟","虽","杂","飞","检","吸","助","升","阳","互","初","创","抗","考","投","坏","策","古","径","换","未","跑","留","钢","曾","端","责","站","简","述","钱","副","尽","帝","射","草","冲","承","独","令","限","阿","宣","环","双","请","超","微","让","控","州","良","轴","找","否","纪","益","依","优","顶","础","载","倒","房","突","坐","粉","敌","略","客","袁","冷","胜","绝","析","块","剂","测","丝","协","重","诉","念","陈","仍","罗","盐","友","洋","错","苦","夜","刑","移","频","逐","靠","混","母","短","皮","终","聚","汽","村","云","哪","既","距","卫","停","烈","央","察","烧","行","迅","境","若","印","洲","刻","括","激","孔","搞","甚","室","待","核","校","散","侵","吧","甲","游","久","菜","味","旧","模","湖","货","损","预","阻","毫","普","稳","乙","妈","植","息","扩","银","语","挥","酒","守","拿","序","纸","医","缺","雨","吗","针","刘","啊","急","唱","误","训","愿","审","附","获","茶","鲜","粮","斤","孩","脱","硫","肥","善","龙","演","父","渐","血","欢","械","掌","歌","沙","著","刚","攻","谓","盾","讨","晚","粒","乱","燃","矛","乎","杀","药","宁","鲁","贵","钟","煤","读","班","伯","香","介","迫","句","丰","培","握","兰","担","弦","蛋","沉","假","穿","执","答","乐","谁","顺","烟","缩","征","脸","喜","松","脚","困","异","免","背","星","福","买","染","井","概","慢","怕","磁","倍","祖","皇","促","静","补","评","翻","肉","践","尼","衣","宽","扬","棉","希","伤","操","垂","秋","宜","氢","套","笔","督","振","架","亮","末","宪","庆","编","牛","触","映","雷","销","诗","座","居","抓","裂","胞","呼","娘","景","威","绿","晶","厚","盟","衡","鸡","孙","延","危","胶","还","屋","乡","临","陆","顾","掉","呀","灯","岁","措","束","耐","剧","玉","赵","跳","哥","季","课","凯","胡","额","款","绍","卷","齐","伟","蒸","殖","永","宗","苗","川","炉","岩","弱","零","杨","奏","沿","露","杆","探","滑","镇","饭","浓","航","怀","赶","库","夺","伊","灵","税","了","途","灭","赛","归","召","鼓","播","盘","裁","险","康","唯","录","菌","纯","借","糖","盖","横","符","私","努","堂","域","枪","润","幅","哈","竟","熟","虫","泽","脑","壤","碳","欧","遍","侧","寨","敢","彻","虑","斜","薄","庭","都","纳","弹","饲","伸","折","麦","湿","暗","荷","瓦","塞","床","筑","恶","户","访","塔","奇","透","梁","刀","旋","迹","卡","氯","遇","份","毒","泥","退","洗","摆","灰","彩","卖","耗","夏","择","忙","铜","献","硬","予","繁","圈","雪","函","亦","抽","篇","阵","阴","丁","尺","追","堆","雄","迎","泛","爸","楼","避","谋","吨","野","猪","旗","累","偏","典","馆","索","秦","脂","潮","爷","豆","忽","托","惊","塑","遗","愈","朱","替","纤","粗","倾","尚","痛","楚","谢","奋","购","磨","君","池","旁","碎","骨","监","捕","弟","暴","割","贯","殊","释","词","亡","壁","顿","宝","午","尘","闻","揭","炮","残","冬","桥","妇","警","综","招","吴","付","浮","遭","徐","您","摇","谷","赞","箱","隔","订","男","吹","乐","园","纷","唐","败","宋","玻","巨","耕","坦","荣","闭","湾","键","凡","驻","锅","救","恩","剥","凝","碱","齿","截","炼","麻","纺","禁","废","盛","版","缓","净","睛","昌","婚","涉","筒","嘴","插","岸","朗","庄","街","藏","姑","贸","腐","奴","啦","惯","乘","伙","恢","匀","纱","扎","辩","耳","彪","臣","亿","璃","抵","脉","秀","萨","俄","网","舞","店","喷","纵","寸","汗","挂","洪","着","贺","闪","柬","爆","烯","津","稻","墙","软","勇","像","滚","厘","蒙","芳","肯","坡","柱","荡","腿","仪","旅","尾","轧","冰","贡","登","黎","削","钻","勒","逃","障","氨","郭","峰","币","港","伏","轨","亩","毕","擦","莫","刺","浪","秘","援","株","健","售","股","岛","甘","泡","睡","童","铸","汤","阀","休","汇","舍","牧","绕","炸","哲","磷","绩","朋","淡","尖","启","陷","柴","呈","徒","颜","泪","稍","忘","泵","蓝","拖","洞","授","镜","辛","壮","锋","贫","虚","弯","摩","泰","幼","廷","尊","窗","纲","弄","隶","疑","氏","宫","姐","震","瑞","怪","尤","琴","循","描","膜","违","夹","腰","缘","珠","穷","森","枝","竹","沟","催","绳","忆","邦","剩","幸","浆","栏","拥","牙","贮","礼","滤","钠","纹","弹","罢","拍","咱","喊","袖","埃","勤","罚","焦","潜","伍","墨","欲","缝","姓","刊","饱","仿","奖","铝","鬼","丽","跨","默","挖","链","扫","喝","袋","炭","污","幕","诸","弧","励","梅","奶","洁","灾","舟","鉴","苯","讼","抱","毁","率","懂","寒","智","埔","寄","届","跃","渡","挑","丹","艰","贝","碰","拔","爹","戴","码","梦","芽","熔","赤","渔","哭","敬","颗","奔","藏","铅","熟","仲","虎","稀","妹","乏","珍","申","桌","遵","允","隆","螺","仓","魏","锐","晓","氮","兼","隐","碍","赫","拨","忠","肃","缸","牵","抢","博","巧","壳","兄","杜","讯","诚","碧","祥","柯","页","巡","矩","悲","灌","龄","伦","票","寻","桂","铺","圣","恐","恰","郑","趣","抬","荒","腾","贴","柔","滴","猛","阔","辆","妻","填","撤","储","签","闹","扰","紫","砂","递","戏","吊","陶","伐","喂","疗","瓶","婆","抚","臂","摸","忍","虾","蜡","邻","胸","巩","挤","偶","弃","槽","劲","乳","邓","吉","仁","烂","砖","租","乌","舰","伴","瓜","浅","丙","暂","燥","橡","柳","迷","暖","牌","纤","秧","胆","详","簧","踏","瓷","谱","呆","宾","糊","洛","辉","愤","竞","隙","怒","粘","乃","绪","肩","籍","敏","涂","熙","皆","侦","悬","掘","享","纠","醒","狂","锁","淀","恨","牲","霸","爬","赏","逆","玩","陵","祝","秒","浙","貌","役","彼","悉","鸭","着","趋","凤","晨","畜","辈","秩","卵","署","梯","炎","滩","棋","驱","筛","峡","冒","啥","寿","译","浸","泉","帽","迟","硅","疆","贷","漏","稿","冠","嫩","胁","芯","牢","叛","蚀","奥","鸣","岭","羊","凭","串","塘","绘","酵","融","盆","锡","庙","筹","冻","辅","摄","袭","筋","拒","僚","旱","钾","鸟","漆","沈","眉","疏","添","棒","穗","硝","韩","逼","扭","侨","凉","挺","碗","栽","炒","杯","患","馏","劝","豪","辽","勃","鸿","旦","吏","拜","狗","埋","辊","掩","饮","搬","骂","辞","勾","扣","估","蒋","绒","雾","丈","朵","姆","拟","宇","辑","陕","雕","偿","蓄","崇","剪","倡","厅","咬","驶","薯","刷","斥","番","赋","奉","佛","浇","漫","曼","扇","钙","桃","扶","仔","返","俗","亏","腔","鞋","棱","覆","框","悄","叔","撞","骗","勘","旺","沸","孤","粘","吐","孟","渠","屈","疾","妙","惜","仰","狠","胀","谐","抛","霉","桑","岗","嘛","衰","盗","渗","脏","赖","涌","甜","曹","阅","肌","哩","厉","烃","纬","毅","昨","伪","症","煮","叹","钉","搭","茎","笼","酷","偷","弓","锥","恒","杰","坑","鼻","翼","纶","叙","狱","逮","罐","络","棚","抑","膨","蔬","寺","骤","穆","冶","枯","册","尸","凸","绅","坯","牺","焰","轰","欣","晋","瘦","御","锭","锦","丧","旬","锻","垄","搜","佛","扑","邀","亭","酯","迈","舒","脆","酶","闲","忧","酚","顽","羽","涨","卸","仗","陪","薄","辟","惩","杭","姚","肚","捉","飘","漂","昆","欺","吾","郎","烷","汁","呵","饰","萧","雅","邮","迁","燕","撒","姻","赴","宴","烦","削","债","帐","斑","铃","旨","醇","董","饼","雏","姿","拌","傅","腹","妥","揉","贤","拆","歪","葡","胺","丢","浩","徽","昂","垫","挡","览","贪","慰","缴","汪","慌","冯","诺","姜","谊","凶","劣","诬","耀","昏","躺","盈","骑","乔","溪","丛","卢","抹","易","闷","咨","刮","驾","缆","悟","摘","铒","掷","颇","幻","柄","惠","惨","佳","仇","腊","窝","涤","剑","瞧","堡","泼","葱","罩","霍","捞","胎","苍","滨","俩","捅","湘","砍","霞","邵","萄","疯","淮","遂","熊","粪","烘","宿","档","戈","驳","嫂","裕","徙","箭","捐","肠","撑","晒","辨","殿","莲","摊","搅","酱","屏","疫","哀","蔡","堵","沫","皱","畅","叠","阁","莱","敲","辖","钩","痕","坝","巷","饿","祸","丘","玄","溜","曰","逻","彭","尝","卿","妨","艇","吞","韦","怨","矮","歇","郊","禄","捻","漠","粹","颠","宏","冤","肪","饥","呵","仙","押","挨","醛","娃","拾","没","佩","勿","吓","讹","侯","恋","夕","锌","篡","戚","淋","蓬","岂","釉","兆","泊","魂","拘","亡","杠","摧","氟","颂","浑","凌","铀","诱","犁","谴","颁","舶","扯","嘉","萌","犹","滋","焊","舌","匹","媳","肺","掠","酿","烹","疲","驰","鸦","窄","辱","狭","朴","遣","菲","奸","韧","辣","拳","秆","卧","醉","竭","茅","墓","矣","哎","艳","敦","舆","缔","雇","尿","葬","履","契","禽","渣","衬","躲","赔","咸","溉","贼","醋","堤","抖","妃","裤","廉","晴","挽","掀","茫","丑","亥","拦","悠","阐","慧","佐","奇","竖","孝","柜","麟","绣","遥","逝","愁","肖","昭","芬","逢","窑","捷","圜","盲","闸","宙","辐","披","账","狼","幽","绸","蜂","慎","餐","酬","誓","惟","叉","弥","址","帜","芝","砌","唉","仆","涛","臭","翠","盒","劫","慨","炳","阖","寂","椒","倘","拓","畏","喉","巾","颈","垦","拚","兽","蔽","芦","乾","爽","窃","谭","挣","崩","模","褐","传","翅","儒","伞","晃","谬","胚","剖","凑","眠","浊","霜","礁","蔑","抄","闯","洒","碑","蓉","耶","猜","蹲","壶","唤","澳","锯","郡","玲","绵","纽","梳","掏","吁","锤","鼠","穴","椅","殷","遮","吵","萍","厌","畜","俱","夸","吕","囊","捧","雌","闽","饶","瞬","郁","哨","凿","朝","俺","浒","茂","肝","勋","盯","籽","耻","菊","滥","稼","戒","奈","帅","鞭","蚕","镁","询","跌","烤","坛","宅","笛","鄂","蛮","颤","棍","睁","鼎","岌","降","侍","藩","嚷","匪","岳","糟","缠","迪","泄","卑","氛","堪","萝","盛","碘","缚","悦","澄","甫","攀","屠","溢","拱","晰","携","朽","吟","菱","谦","凹","俊","芒","盼","婶","艘","酰","趁","唇","挫","羞","浴","疼","萎","肴","愚","肿","刨","绞","枢","嫁","慕","舱","铲","苹","豫","谕","迭","潘","顷","翁","榜","匠","欠","茬","畴","胃","沾","踪","弊","哼","鹏","歧","桐","沃","悼","惑","溃","蔗","荐","潭","孢","露","诊","庸","聪","嫌","厨","庞","祁","钳","肆","梭","赠","崖","篮","颖","甸","藻","捣","且","撕","诏","贞","赐","慈","炕","胖","兹","差","琼","锈","汛","卓","棵","馈","挠","灶","婴","蒂","肤","衫","沥","仑","勉","沪","逸","蜜","浦","嗓","晕","膏","祭","赢","艾","扮","鹅","怜","蒲","兔","孕","呖","蘖","挪","淑","谣","惧","廊","缅","俘","骄","膀","陡","宰","诞","峻","恼","腺","猎","涡","夷","愉","魔","铵","葛","贾","似","荫","哟","脊","钞","苛","锰","椭","镶","杏","溴","倚","滞","会","氓","捏","斩","傲","匆","僵","卤","烫","衍","榨","拢","裸","屑","咽","坊","舅","渴","翔","邪","拄","窖","猫","砌","钦","媒","脾","勺","柏","栅","噪","昼","耿","扁","辰","秤","得","贩","糕","梁","昙","衷","宦","扔","哇","诈","嘱","藤","卜","冈","悔","廓","皂","拐","氰","杉","玛","矢","寓","瓣","罕","垮","笋","淘","衔","称","恭","喇","帕","桉","秉","帘","铭","蛇","摔","斋","叭","帆","裸","俭","瘤","篷","砸","肢","辟","脖","瞪","暑","卜","竿","歼","笙","酮","蕴","哗","瞎","喀","刃","楔","喘","枚","嵌","挝","厢","粤","甩","拴","膝","恳","腕","娓","熄","锚","忌","愧","哦","荆","圃","骚","丸","蒜","毯","弗","俯","鹿","梢","屯","衙","轿","贱","垒","谅","踢","哑","滔","渥","饷","泳","棕","熬","搁","腈","梨","吻","樱","奠","捆","姨","柏","聘","惕","郓","绑","冀","裹","酥","寡","彦","稠","啡","钝","汝","擅","汰","鳙","埔","敞","嘿","逊","栋","谨","咖","鲤","雀","佣","庵","葫","贿","鳞","拼","搏","谎","塌","忉","腻","戊","怖","坟","禾","刹","嘻","桔","坎","拇","煽","狮","痒","曾","梗","寇","鹰","烛","哄","莽","雯","胳","龟","亟","糠","泌","坪","傻","什","喻","渊","蚌","跪","巷","涅","钊","譬","蕊","膛","侮","奕","枕","辫","况","扼","郝","寥","凄","厦","腥","钧","耦","蹄","戥","屁","诵","匈","桩","钓","涵","倦","袍","抒","屿","蹈","忿","敷","虹","聊","嗣","尉","灿","糙","蹬","嗯","姬","狡","笨","辜","僧","茨","讽","翰","枉","岐","枣","崭","焚","咕","猴","揽","涝","耍","趟","汹","咋","傍","镀","给","爵","虏","劈","璋","踩","瞅","迄","昔","汞","呱","诡","魄","祺","嘲","惶","赃","癌","咐","歉","扳","鄙","庐","聂","便","芡","躯","贬","煌","拧","隋","襄","淤","宠","炊","滇","謇","懒","栓","佑","憾","骆","裙","猖","兜","孵","痼","盥","曝","泣","絮","韵","眷","旷","噢","参","栖","盏","鳌","溅","煎","校","榴","暮","琪","淆","陛","巢","哒","吼","槐","唧","其","沛","乞","蜀","蜇","赚","捍","铰","幂","尧","咒","耽","叮","褂","焕","煞","雹","搓","釜","铬","拣","募","淹","瑰","鲢","茄","灼","邹","躬","觉","娇","焉","彰","鹤","琳","沦","畔","惹","庶","毙","皖","邢","禹","渍","绷","窜","翘","淫","箪","陌","膊","鞑","咳","玫","巫","拂","蕉","澜","赎","绥","锄","囱","赌","颊","缕","寅","躁","稚","庚","苟","氦","魁","珊","蜕","蛭","酌","逗","闺","蔓","撇","豌","朕","缉","襟","镍","桅","荧","侄","卒","佃","瞿","娶","饪","耸","乍","靶","痴","靖","扛","筐","韶","嚣","崔","蓿","岔","氘","娥","剿","霖","喃","搪","雍","裳","撰","豹","骏","慷"] + static def totalCount = new AtomicLong() - //"福州":"fz|304","合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319","成都":"cd|102", -// "青岛":"qd|122", -// "重庆":"cq|37","石家庄":"sjz|241","广州":"gz|3","深圳":"sz|4","东莞":"dg|413","佛山":"fs|222","杭州":"hz|79","武汉":"wh|158", -// "上海":"sh|2", - def cityList = [ "北京":"bj|1","天津":"tj|18","南京":"nj|172", - "嘉兴":"jx|497","大连":"dl|147","乌鲁木齐":"xj|984","太原":"ty|740","兰州":"lz|952","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362","常州":"cz|463", - "南通":"nt|394","吉林":"jl|700","中山":"zs|771","台州":"tz|403","东营":"dy|623","三亚":"sanya|2422", - "呼和浩特":"hu|811","济南":"jn|265","温州":"wz|330","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096", - "临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385","盐城":"yancheng|613","烟台":"yt|228","信阳":"xy|8694","南充":"nanchong|2378","洛阳":"luoyang|556","南阳":"ny|592","厦门":"xm|606", + //"福州":"fz|304","蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158", +// "兰州":"lz|952","合肥":"hf|837", "乌鲁木齐":"xj|984","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362", +// "芜湖":"wuhu|2045","东莞":"dg|413","常州":"cz|463","佛山":"fs|222", +// "南通":"nt|394","东莞":"dg|413","吉林":"jl|700","中山":"zs|771","三亚":"sanya|2422","阜阳":"fy|2325", +// "呼和浩特":"hu|811","济南":"jn|265","温州":"wz|330","台州":"tz|403","东营":"dy|623","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096", +// "临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385", + def cityList = ["北京":"bj|1","上海":"sh|2","深圳":"sz|4","石家庄":"sjz|241","太原":"ty|740","成都":"cd|102","杭州":"hz|79","重庆":"cq|37","盐城":"yancheng|613","烟台":"yt|228","信阳":"xy|8694","南充":"nanchong|2378","洛阳":"luoyang|556","南阳":"ny|592","厦门":"xm|606", "宜昌":"yc|858","张家口":"zjk|3328","昆明":"km|541","柳州":"liuzhou|7133","绵阳":"mianyang|1057", "泉州":"qz|291","济宁":"jining|450","贵阳":"gy|2015","连云港":"lyg|2049","邯郸":"hd|572", "淮北":"huaibei|9357","滁州":"chuzhou|10266", "马鞍山":"mas|2039","铜陵":"tongling|10285","宣城":"xuancheng|5633","亳州":"bozhou|2329","黄山":"huangshan|2323", @@ -199,7 +199,7 @@ class Esf58Crawl extends BaseBatch{ distMap.put(city, districts) addCrawlKeys(city); - addDetailContent(city, url); + addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); // Thread.sleep(1000*60*10) } @@ -216,13 +216,13 @@ class Esf58Crawl extends BaseBatch{ def districtName = districts[i].name println "${city}【${i}/${districts.size()}】 "+districtName for(def j=1;j<=districtName.length()-1;j++) { - if(j<=2){ +// if(j<=3){ def keyword = districtName.toString().substring(0, j); - def keyStr = city+"##"+keyword; + def keyStr = keyword; if(!useDistKeys.contains(keyStr)){ useDistKeys << keyStr } - } +// } } } useDistKeys.sort {a,b-> @@ -235,16 +235,17 @@ class Esf58Crawl extends BaseBatch{ - def addDetailContent(def city, def url){ + def addDetailContent(def city, def url, def cityid){ final CountDownLatch latch = new CountDownLatch(useDistKeys.size()) for(def k=0;k0){ + def datas = jsonObj.data + for(def d : datas){ + def obj = findByParams(city, d.areaName, d.name); + def price = null + if(d.price){ + price = d.price.toInteger() + } + if(obj){ + DBObject udata2 = new BasicDBObject() + udata2.put("avg_price", price); + udata2.put("c_date", new Date()); + println "[update] "+city + " "+ d.name + thirdPartyColl.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata2)); + }else{ + + println "[save] "+city + " "+ d.name + saveThirdParty(["url":d.listName,"city":city,"region":d.areaName,"block":null,"name":d.name,"address": d.address, "avg_price": price]); + } + } + } + }else{ + println "【"+keyword +"】 无结果" + } + + } + } + def addSearchContent(city, url, keyword){ def flag = true; @@ -416,7 +453,7 @@ class Esf58Crawl extends BaseBatch{ obj.put("s_date", SHORT_DATE); obj.put("source", SOURCE); obj.put("addres", metadata.address); - obj.put("avg_price", metadata.avg_price.toInteger()); + obj.put("avg_price", metadata.avg_price); obj.put("c_date", new Date()); println obj.toString() thirdPartyColl.save(obj); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 3993f6663..1aae0a14c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -17,8 +17,8 @@ class SynchDistrictPrice { def districtDetailColl = CrawalDBUtil.getDistrictDetailColl(); // def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房", // "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"] - def sourceMap = ["58.com":"58同城","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多", - "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"] + //"fang.com":"搜房", + def sourceMap = ["fang.com":"搜房"] static main(args) { def task = new SynchDistrictPrice() task.run()