From c4131ee835e14eeff26d6e57a8d2a49509809ac1 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 29 Jun 2022 17:45:57 +0800 Subject: [PATCH] =?UTF-8?q?=E7=A6=8F=E5=B7=9E=E5=90=8C=E6=AD=A5=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E4=BF=AE=E6=94=B9=EF=BC=8C=E5=A2=9E=E5=8A=A0=E4=BA=BA?= =?UTF-8?q?=E5=B7=A5=E5=88=A0=E9=99=A4=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../DealAndThirdPriceDataProcess.groovy | 6 +-- .../script/ccbevaluation/LoadGuide.groovy | 17 ++++++- .../fangjia/script/ccbevaluation/Test.groovy | 48 ++++++++++--------- .../script/commData/CommonDatautils.groovy | 21 ++++++++ .../fuzhou/DistrictInfoExportBatch.groovy | 9 ++-- .../fuzhou/DistrictPriceExportBatch.groovy | 48 +++++++------------ .../com/fangjia/script/gj/TestFjtLog.groovy | 4 +- .../script/gj/crawl/AnjukeHouseCrawl.groovy | 14 ++++-- .../script/gj/crawl/AnjukeSendMq.groovy | 2 +- .../gj/location/DistrictMapMatch.groovy | 48 ++++++++++++++++++- .../fangjia/script/gj/paimai/DataUtil.groovy | 1 + .../script/gj/paimai/PaimaiTest.groovy | 34 +++++++++++-- .../script/thirdPriceCrawl/DataStat.groovy | 6 +-- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 20 +++++--- 14 files changed, 194 insertions(+), 84 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index c8697c956..f35522733 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -373,10 +373,10 @@ class DealPriceDataProcess { def citys = CcbCommonUtils.citys def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def publishTime = '2021-10-01' - def month = 202204 + def month = 202205 def rows = [] for(def city : citys) { - def rets = findPeGteThirdParty(city, 202205) + def rets = findPeGteThirdParty(city, 202206) def total = 0 println city +" "+rets.size() for(def m : rets){ @@ -388,7 +388,7 @@ class DealPriceDataProcess { } def findPeGteThirdParty(cityName, yearmonth){ - def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>50" + def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>30" def rets = [] sqlInstance236.eachRow(sql,{r-> def map = [:] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy index b0476927d..9985dd73f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy @@ -23,7 +23,8 @@ class LoadGuide { } def run(){ - addPersonGuided("D:\\数据\\建行\\人工指导价更新.xlsx") +// addName("D:\\数据\\建行\\房产估值\\人工指导价更新.xlsx") + addPersonGuided("D:\\数据\\建行\\房产估值\\人工指导价更新.xlsx") // match("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx") // saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞指导价218.xlsx") @@ -38,6 +39,20 @@ class LoadGuide { } + def addName(filename){ + ExcelUtils.multiThreadingModify(filename, { d -> + if(d['A']){ + def dist = CommonDatautils.findDistrictById(d['A'].toString()) + if(dist){ + d['B'] = dist.city + d['C'] = dist.region + d['D'] = dist.name + } + } + + }) + } + def addPersonGuided(filename){ ExcelUtils.modify(filename, { d -> def city = d['B'] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 72bc95878..41a0fe686 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -45,7 +45,9 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl) - +def url = "https://sh.anjuke.com" +def s = DataHandleUtils.findMatchPattern(name, "https://\\w+.anjuke.com") +println s //def districts = CommonDatautils.findAllFiledDistrict("district", "南宁") //for(def d : districts){ @@ -324,28 +326,28 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def // //},0) -ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\删除重复小区.xlsx",{d-> - if(d['A']){ -// def region = d['B'] -// def newRegion = RegionData.formatRegion(d['A'], region) -// if(newRegion) { -// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") - def dist = CommonDatautils.findDistrictById(d['A']) - if(dist){ - def udata = new BasicDBObject() - udata.put("name_cat", 11) - udata.put("status", 1) - udata.put("m_date",new Date()) - - def q = new BasicDBObject() - q.put("_id", dist._id) - DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) - }else{ - println d - } -// } - } -},0) +//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\删除重复小区.xlsx",{d-> +// if(d['A']){ +//// def region = d['B'] +//// def newRegion = RegionData.formatRegion(d['A'], region) +//// if(newRegion) { +//// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") +// def dist = CommonDatautils.findDistrictById(d['A']) +// if(dist){ +// def udata = new BasicDBObject() +// udata.put("name_cat", 11) +// udata.put("status", 1) +// udata.put("m_date",new Date()) +// +// def q = new BasicDBObject() +// q.put("_id", dist._id) +// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +// }else{ +// println d +// } +//// } +// } +//},0) //def citys = CcbCommonUtils.citys //def rets = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index e374d233a..38d9b23b3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -239,6 +239,27 @@ class CommonDatautils { } + public static def findRemoveDistricts(def city, def cat, def fields, def m_date){ + DBObject q = new BasicDBObject(); + q.put("status", 1); + q.put("cat", cat); + q.put("city", city); + q.put("m_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(m_date))) + DBObject keys = new BasicDBObject(); + if(fields){ + for(def field : fields){ + keys.put(field, 1); + } + } + + DSCursor cursor = getDataStoreConnection().find(q, keys).batchSize(2000) + List rets = [] + for(def obj : cursor){ + rets << obj + } + return rets; + } + /** * 获取城市中小区的经纬度 * @param city diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy index 9934cb1c7..ce903527a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy @@ -68,14 +68,17 @@ class DistrictInfoExportBatch extends BaseBatch{ def createDatas(def city){ def result = [] def districts = CommonDatautils.findAllDistricts(city, "district", null); + def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01"); + districts.addAll(removeDists) def lastDate = DateUtils.getLastDayOfMonth(new Date(), -1) def startDate = DateUtils.getFirstDayOfMonth(lastDate); def fzAddrMap = getFzAddress(); for(def rMap : districts){ - def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate) - if(dbo){ + //价格没有也提供 +// def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate) +// if(dbo){ def row = [] row << rMap.get("_id").toString() row << rMap.city @@ -112,7 +115,7 @@ class DistrictInfoExportBatch extends BaseBatch{ } //row << processAlias(rMap.addr_alias) result << row - } +// } } return result } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy index bb7b11f46..dd9cd246b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy @@ -88,6 +88,9 @@ class DistrictPriceExportBatch extends BaseBatch{ def createDatas(def city){ def districts = CommonDatautils.findAllDistricts(city, "district", null); + def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01"); + districts.addAll(removeDists) + def endDateStr = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-10" def dataStr = DateUtils.add(endDateStr, -30) def startDate = DateUtils.parseFromTightYYYYMMDD(dataStr) @@ -105,9 +108,10 @@ class DistrictPriceExportBatch extends BaseBatch{ for(def rMap : districts){ def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate) + def curPrice = 0 if(dbo){ def lastMonthPrice = lastMonthPriceMap.get(rMap.get("_id").toString()) - def curPrice = dbo.get("avg_price") + curPrice = dbo.get("avg_price") if(lastMonthPrice && curPrice){ def markup = (curPrice - lastMonthPrice)*100/lastMonthPrice if(Math.abs(markup)>3){ @@ -120,36 +124,20 @@ class DistrictPriceExportBatch extends BaseBatch{ curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice) } } - - def m = [] - m << dbo.get("_id").toString()//均价ID - m << rMap.get("_id").toString()//楼盘ID - m << rMap.city - m << rMap.region - m << rMap.name - m << curPrice -// def lastMonthPrice = map2.get(rMap.city + "##" + rMap.region+"##"+rMap.name) -// if(lastMonthPrice && dbo.get("avg_price")){ -// lastMonthPrice = lastMonthPrice.toInteger() -// def markup = (dbo.get("avg_price")-lastMonthPrice)*100.0/lastMonthPrice -// if(Math.abs(markup)>10){ -// def random = 0 -// if(dbo.get("avg_price")-lastMonthPrice>=0){ -// random = com.fangjia.core.util.RandomUtil.random(1, 9) -// }else{ -// random = com.fangjia.core.util.RandomUtil.random(-9, 0) -// } -// def newPrice = lastMonthPrice*random/100 + lastMonthPrice -// m << newPrice.toInteger() -// }else{ -// m << dbo.get("avg_price") -// } -// }else{ -// m << dbo.get("avg_price") -// } - m << curDate.toInteger(); - result << m } + def m = [] + if(curPrice){ + m << dbo.get("_id").toString()//均价ID + }else{ + m << "" + } + m << rMap.get("_id").toString()//楼盘ID + m << rMap.city + m << rMap.region + m << rMap.name + m << curPrice + m << curDate.toInteger(); + result << m } return result } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TestFjtLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TestFjtLog.groovy index 82bb78ccd..5dac60e11 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TestFjtLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TestFjtLog.groovy @@ -10,11 +10,11 @@ import java.text.SimpleDateFormat class TestFjtLog { - public static final DBCollection COLL = MongoUtils.getCollectionByName("fangjiatong_log", "fjt_select_log", "192.168.0.42", 52002) + public static final DBCollection COLL = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.42", 52002) static main(args) { def q = new BasicDBObject() - q.put("cat","土地") +// q.put("cat","土地") def dbcursor = COLL.find(q).sort(new BasicDBObject("_id",-1)) def map = [:] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy index a12d563ea..81056b29d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy @@ -25,11 +25,11 @@ class AnjukeHouseCrawl { System.setProperty("http.nonProxyHosts", "localhost"); WebDriver driver = new ChromeDriver() - def href = "https://nanjing.anjuke.com/community/view/634969" -// def doc = getDocument(href, driver) - def doc = ClaABuYun.getResponse(href) -// def newUrl = href.substring(0, href.indexOf("?")) - sendMq(href, doc, "district_detail_anjuke") + def href = "https://sh.esf.fang.com/loupan/1210043376.htm" + def doc = getDocument(href, driver) +// def doc = ClaABuYun.getResponse(href) +// def newUrl = href.substring(0, hrefs.indexOf("?")) + sendMq(href, doc, "district_detail_fang") // for(def i=1;i<50;i++){ // def url = "https://fz.anjuke.com/sale/guloue/p${i}/" @@ -201,6 +201,10 @@ class AnjukeHouseCrawl { if(!els){ els = JQueryUtils.select(".tab-cont-right", doc) } + + if(!els){ + els = JQueryUtils.select(".village_info", doc) + } if(els && els.size()>0){ stop = true; }else{ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeSendMq.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeSendMq.groovy index 498fc87bd..19598718f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeSendMq.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeSendMq.groovy @@ -21,7 +21,7 @@ class AnjukeSendMq { def q = new BasicDBObject() def dbcursor = collection.find(q).limit(30) for(def d : dbcursor){ - sendMq(d.url, d.content, "district_detail_anjuke") + sendMq(d.url, d.content, "community_anjuke") } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictMapMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictMapMatch.groovy index 25d718caf..b9b0c36d8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictMapMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictMapMatch.groovy @@ -3,6 +3,7 @@ package com.fangjia.script.gj.location import com.fangjia.core.util.GeoUtils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.gj.paimai.DataUtil import com.fangjia.script.util.CosineSimilarAlgorithm import com.fangjia.script.util.ExcelUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils @@ -13,7 +14,9 @@ class DistrictMapMatch { def task = new DistrictMapMatch() // task.mapMatch() // task.mapMatch1() - task.distince() +// task.distince() + + task.mapMatch2() } def distince(){ @@ -93,6 +96,49 @@ class DistrictMapMatch { } + def mapMatch2(){ + def index = 0 + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\二期\\待入库小区(1).xlsx",{ d-> + if(!d['M']){ + index++ + def rets = OpenApiUtils.getGaoDeSearchLocation(d['C'],d['J'],d['E']) + def map = findStandardObject2(d['C'],d['J'],d['E'], d['A'], rets); +// if (!map || !map.lat) { +// rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']); +// map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets); +// } + if(map){ + d['K'] = map.id + if(map.adname){ + d['L'] = map.adname + }else if(map.region){ + d['L'] = map.region + } + d['M'] = map.name + d['N'] = map.address + + d['O'] = map.lng + d['P'] = map.lat + if(map.type){ + d['Q'] = map.type + }else if(map.estateType){ + d['Q'] = map.estateType + } + d['R'] = map.score + + if(d['O'] && d['AM']){ + def location = d['AM'] + def lng = location.toString().substring(location.toString().indexOf(": ")+2, location.toString().indexOf(",")) + def lat = location.toString().substring(location.toString().lastIndexOf(": ")+2, location.toString().indexOf("}")) + def distance = GeoUtils.calculateDistance(d['O'].toDouble(), d['P'].toDouble(), lng.toDouble(), lat.toDouble()) + d['S'] = distance + } + } + } + },0) + } + + def findStandardObject2(def city, def region, def name, def address, def rets) { if (rets) { def bestObj = [:] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataUtil.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataUtil.groovy index 7e81c3afd..270e44a97 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataUtil.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataUtil.groovy @@ -308,4 +308,5 @@ class DataUtil { return s } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy index 006b8a875..a7523830b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy @@ -1,6 +1,7 @@ package com.fangjia.script.gj.paimai import com.alibaba.fastjson.JSON +import com.fangjia.core.RegionData import com.fangjia.core.util.DataUtil import com.fangjia.script.crawl.JQueryUtils import com.fangjia.script.util.MongoUtils @@ -32,7 +33,20 @@ class PaimaiTest { // task.stat() // task.updatePaimai() // task.updateFieldPaimai() - task.statCity() +// task.statCity() + task.jsStat() + } + + def jsStat(){ + def q = new BasicDBObject() + q.put("city","苏州") + def dbcursor = paimai43_COLL.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def rets = [] + for(def db : dbcursor){ + def m = [:] + + } + } def statCity(){ @@ -60,16 +74,26 @@ class PaimaiTest { def updateFieldPaimai(){ def q = new BasicDBObject() - q.put("city", new BasicDBObject('$in',["重庆","重庆市"])) +// q.put("city", new BasicDBObject('$in',["重庆","重庆市"])) def index = 0 def dbcursor = paimai43_COLL.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) for(def obj : dbcursor) { index++ println index - if(obj.address && obj.address.contains("可贷款")){ - obj.put("address", obj.address.substring(0,obj.address.indexOf("可贷款"))) - paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj)) + def oldCity = obj.city + if(oldCity){ + def newCity = RegionData.formatCity(oldCity) + if(newCity && newCity!=oldCity){ + obj.put("city", newCity) + println index +" "+ oldCity +" "+newCity + paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj)) + + } } +// if(obj.address && obj.address.contains("可贷款")){ +// obj.put("address", obj.address.substring(0,obj.address.indexOf("可贷款"))) +// paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj)) +// } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index ec9d7f64b..1b7bbc303 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -40,11 +40,11 @@ class DataStat { def run(){ - def source = "贝壳找房"// - def citys = findCrawlCity(source, 202202); + def source = "Q房"// + def citys = findCrawlCity(source, 202204); for(def city : citys){ def count1 = findCount(source, city, 202204); - def count2 = findCount(source, city, 202205); + def count2 = findCount(source, city, 202203); println city +" "+count1+" "+count2 // println count1 // println "--- ----" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 22a2ab033..a95c6332f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -33,18 +33,18 @@ class SynchDistrictPrice { esSourceMap.each { k, v -> for(def city : citys){ - def rets = findAllEsData(city, k, v) + def rets = findAllEsData(city, v) println city +" "+rets.size() - synchData(city, rets, v) + synchData(city, rets, "贝壳找房") } } } - def findAllEsData(source){ + def findAllEsData(city,source){ def names = new HashSet() def rets = [] - ElasticUtils.executeSqlQuery("select * from community where source='贝壳'", {d-> + ElasticUtils.executeSqlQuery("select * from community where source='贝壳' and city='${city}'", {d-> def s = d.city +"#"+d.region+"#"+d.name if(!names.contains(s)){ names << s @@ -56,7 +56,7 @@ class SynchDistrictPrice { data.put("avg_price", d.avg_price) data.put("source", source) data.put("s_date", SHORT_DATE) - rets << d + rets << data } }) return rets; @@ -132,8 +132,14 @@ class SynchDistrictPrice { def obj= findIsExists(r.city, r.region, r.name, SHORT_DATE, source) if(obj){ updateIndex++ - thirdPartyColl.remove(new BasicDBObject('_id',obj._id)) - thirdPartyColl.save(r) +// thirdPartyColl.remove(new BasicDBObject('_id',obj._id)) +// thirdPartyColl.save(r) + + def udata = new BasicDBObject() + udata.put("avg_price", r.avg_price) + udata.put("m_date",new Date()) + thirdPartyColl.update(new BasicDBObject('_id',obj._id), udata) + }else{ try{ addIndex++