From f59186f93a8dccd8e64bf611b7ae666978bcc8c3 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 9 Aug 2022 16:37:16 +0800 Subject: [PATCH] update --- .../ccbevaluation/BigdataForsaleBatch.groovy | 2 +- .../ronghe/BigdataCommunityGJRHBatch.groovy | 2 +- .../fuzhou/DistrictPriceExportBatch.groovy | 2 +- .../fuzhou/ThirdPriceExportBatch.groovy | 124 ++++++++++++++---- .../gj/district/FangjiaClientUtils.groovy | 6 + .../gj/paimai/BeiKeCrawlByChrome.groovy | 7 +- .../gj/paimai/LianjiaCrawlByChrome.groovy | 6 +- .../script/gj/paimai/PaimaiDataSend.groovy | 83 +++++++++++- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 4 +- 9 files changed, 199 insertions(+), 37 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 7088dea02..12a02fabb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -450,7 +450,7 @@ class BigdataForsaleBatch extends BaseBatch{ DBObject seawater = new BasicDBObject(map) if(estate_type2 && estate_type2=="普通住宅"){ - if(seawater.estate_type&& seawater.estate_type.contains("商住")){ + if(seawater.estate_type && (seawater.estate_type.contains("商住") || seawater.estate_type.contains("商业") || seawater.estate_type.contains("商务"))){ println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type continue; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy index dc70af53b..be808deaa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy @@ -71,7 +71,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { // } // } - def distMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["address","build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"]) + def distMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["address","build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge"]) for(def dist : districts){ def obj = distMap.get(dist.commid)//CommonDatautils.findDistrictFieldById(dist.commid,["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"]) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy index c38bb7346..276c86cec 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy @@ -112,7 +112,7 @@ class DistrictPriceExportBatch extends BaseBatch{ if(dbo){ def lastMonthPrice = lastMonthPriceMap.get(rMap.get("_id").toString()) curPrice = dbo.get("avg_price") - if(lastMonthPrice && curPrice){ + if(lastMonthPrice && curPrice && dbo.value_type!=1){ def markup = (curPrice - lastMonthPrice)*100/lastMonthPrice if(Math.abs(markup)>3){ def randMarkup = nextDouble(0, 3) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy index 2cceebe1e..9e3a6dd7e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy @@ -23,6 +23,9 @@ class ThirdPriceExportBatch extends BaseBatch{ static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com") + + def charset = "GBK" static main(args) { @@ -55,8 +58,12 @@ class ThirdPriceExportBatch extends BaseBatch{ def districts = CommonDatautils.findAllDistricts(city, "district", ["city","region","name"]); def result = [] def index = 0 + def lastMonthPriceMap = findFzDistrictStat("福州", curDate) + for(def dist : districts){ - def rets = findThirdPrice(dist.city, dist.region, dist.name) + def key = dist.city +"##"+ dist.region+"##"+ dist.name + def rets = findThirdPrice(dist.city, dist.region, dist.name) + def curPrice = lastMonthPriceMap.get(key) index ++ if(rets){ for(def o : rets){ @@ -68,46 +75,84 @@ class ThirdPriceExportBatch extends BaseBatch{ row << dist.region row << dist.name row << o.s_date - row << o.zhugezhaofang_esf_price - if(o.zhugezhaofang_esf_price){ + def zgf_price = getLastThirdPrice(o.zhugezhaofang_esf_price, curPrice) + row << zgf_price + if(zgf_price){ add = true; } - row << o.anjuke_esf_price - if(o.anjuke_esf_price){ - add = true; - } - row << o.soufang_esf_price - if(o.soufang_esf_price){ +// if(o.zhugezhaofang_esf_price){ +// add = true; +// } + def ajk_price = getLastThirdPrice(o.anjuke_esf_price, curPrice) + row << ajk_price + if(ajk_price){ add = true; } +// if(o.anjuke_esf_price){ +// add = true; +// } - row << o.lianjia_esf_price - if(o.lianjia_esf_price){ + def sf_price = getLastThirdPrice(o.soufang_esf_price, curPrice) + row << sf_price + if(sf_price){ add = true; } +// if(o.soufang_esf_price){ +// add = true; +// } + def lj_price = getLastThirdPrice(o.lianjia_esf_price, curPrice) + row << lj_price + if(lj_price){ + add = true; + } +// if(o.lianjia_esf_price){ +// add = true; +// } - row << o.cityhouse_esf_price - if(o.cityhouse_esf_price){ + def city_price = getLastThirdPrice(o.cityhouse_esf_price, curPrice) + row << city_price + if(city_price){ add = true; } +// if(o.cityhouse_esf_price){ +// add = true; +// } - row << o.fangtan_esf_price - if(o.fangtan_esf_price){ - add = true; - } - row << o.fangdoudou_esf_price - if(o.fangdoudou_esf_price){ - add = true; - } - row << o.maitian_esf_price - if(o.maitian_esf_price){ + def ft_price = getLastThirdPrice(o.fangtan_esf_price, curPrice) + row << ft_price + if(ft_price){ add = true; } +// if(o.fangtan_esf_price){ +// add = true; +// } - row << o.get("58tc_esf_price") - if(o.get("58tc_esf_price")){ + def fdd_price = getLastThirdPrice(o.fangdoudou_esf_price, curPrice) + row < JsonToMap(JSONObject json) { Map columnValMap = new HashMap(); Set jsonKeys = json.keySet(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index ec874a945..cbeaf0ef9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -32,6 +32,7 @@ public class BeiKeCrawlByChrome { def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM") static main(args) { def task = new BeiKeCrawlByChrome(); @@ -160,7 +161,7 @@ public class BeiKeCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2022-05-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd") if(dealDate && dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e) @@ -375,13 +376,13 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = e.attr("href") -// if(e.text().contains("福州")){ + if(e.text().contains("福州")){ def cityMap = [:] cityMap.city = e.text() cityMap.url = "https:"+href list << cityMap //cityMap.put(e.text(), "https:"+href) -// } + } } } return list diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 0388e7fe0..86da3466b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -34,7 +34,7 @@ public class LianjiaCrawlByChrome { static main(args) { def task = new LianjiaCrawlByChrome(); -// task.run() + task.run() // task.run1() task.addDealRegion(); task.standardRegion() @@ -238,7 +238,7 @@ public class LianjiaCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2022-05-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-06-20", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e) @@ -442,7 +442,7 @@ public class LianjiaCrawlByChrome { def cityDoc = Jsoup.parse(driver.getPageSource()); def cityEles = JQueryUtils.select(".city_province a", cityDoc) def cityMap = [:] - def citys= ["安庆"] + def citys= ["福州"] if(cityEles){ for(def e : cityEles) { def href = e.attr("href") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy index 4a2ad0792..7a314a59c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy @@ -2,6 +2,7 @@ package com.fangjia.script.gj.paimai import com.alibaba.fastjson.JSON import com.alibaba.fastjson.JSONObject +import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.temp.shenjian.util.MessageUtils import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils @@ -32,13 +33,14 @@ class PaimaiDataSend { static main(args) { def task = new PaimaiDataSend() - task.jdRun() +// task.jdRun() // task.aliZcRun() // task.aliZcRun2() // task.aliFpRun() // task.rmfysszcRun() // task.checkUrl() + task.paimaiSendToEs() } @@ -298,6 +300,85 @@ class PaimaiDataSend { ali_zc_coll.update(new BasicDBObject("_id", d.get("_id")), new BasicDBObject('$set', udata)) } } + + def paimaiSendToEs(){ + def rets = getPaimaiDatas() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(rets.size()) + for(def i=0;i'2022-06-15'", {d-> + ElasticUtils.executeSqlQuery("select * from community where source='${esSource}' and city='${city}' and m_date>'2022-08-01'", {d-> def s = d.city +"#"+d.region+"#"+d.name if(!names.contains(s)){ names << s