From a3ec34c62a4a2207dd199da7fc2d8124c317af1c Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 23 Sep 2021 10:07:54 +0800 Subject: [PATCH] update --- .../ccbevaluation/CcbCommonUtils.groovy | 40 ++++++++++++++++++- .../script/commData/CommonDatautils.groovy | 22 ++++++++++ .../fangjia/script/gj/GaoDeMapCrawl.groovy | 19 ++++++--- .../script/thirdPriceCrawl/Esf58Crawl.groovy | 17 ++++---- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 5 ++- 5 files changed, 88 insertions(+), 15 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 0b413551a..f52ccdf19 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -103,6 +103,13 @@ class CcbCommonUtils { alias << dist.name } + if(dist.name.endsWith("弄小区")){ + def name = dist.name.replace("弄小区","弄") + if(!alias.contains(name)){ + alias << name + } + } + if(alias){ q.put("district_name", new BasicDBObject('$in', alias)) } @@ -131,6 +138,37 @@ class CcbCommonUtils { return lists } + public static def findTradesDetailByAddress(dist, startDate, endDate, startPrice, endPrice){ + def q = new BasicDBObject() + q.put("city", dist.city) + q.put("region", dist.region) + q.put("area", new BasicDBObject('$gte', 20)) + q.put("district_name", dist.address) + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + def dbcursor = DEAL_PRICE_COLL.find(q) + def total = dbcursor.count() + def rets = new HashSet() + def lists = [] + for(def d : dbcursor){ + if(startPrice && d.avg_price>=startPrice && d.avg_price<=endPrice){ + def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd") + if(!rets.contains(key)){ + rets << key + d.dist_id = dist._id.toString() + d.address = dist.address + d.master_name = dist.name + d.build_type = dist.build_type + d.build_structure = dist.build_structure + d.house_roperty = dist.house_roperty + d.property_life = dist.property_life + d.estate_type2 = dist.estate_type2 + lists << d + } + } + } + return lists + } + public static def findTrades(dist, startDate, endDate){ def q = new BasicDBObject() @@ -187,7 +225,7 @@ class CcbCommonUtils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name", "莱兴公寓") +// q.put("name", "周家湾") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 42d3641f3..f98fd8db7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1231,6 +1231,28 @@ class CommonDatautils { return ret; } + + + public static def findDistrictByAddress2(def city, def region, def address, def cat){ + DBObject q = new BasicDBObject(); + if(region){ + q.put('region', region) + } + + q.put('city',city) + q.put('cat', cat) + q.put('visible',0) + q.put("address", address); + + + def dsCursor = getDataStoreConnection().find(q); + def ret = []; + for(def obj : dsCursor){ + ret << obj; + } + + return ret; + } public static def getCityRegions(city){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy index ddc92508f..520a1bdd0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy @@ -119,10 +119,11 @@ class GaoDeMapCrawl extends BaseBatch{ def run(){//,"西安","重庆" //def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] - def citys =["上海","南京","苏州","无锡","扬州","常州","徐州","镇江","南通","泰州","连云港","盐城","淮安","宿迁","杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"] + def citys = ["上海","南京","苏州","无锡","扬州","常州","徐州","镇江","南通","泰州","连云港","盐城","淮安","宿迁","杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"] def allcitys = CommonUtils.findAllCity() def totalCount = 0; for(def city : allcitys){ + if(!citys.contains(city)){ def stop = false while(stop == false){ @@ -135,6 +136,7 @@ class GaoDeMapCrawl extends BaseBatch{ } } } + } } @@ -146,7 +148,11 @@ class GaoDeMapCrawl extends BaseBatch{ q.put(prefix+"status", new BasicDBObject(QueryOperators.EXISTS, false)) // q.put("status", "失败") // q.put("traffic_status", new BasicDBObject(QueryOperators.EXISTS, false)) - def results = BOUNDS_COLL.find(q).limit(2000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def dbcursor = BOUNDS_COLL.find(q).batchSize(3000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def results = [] + for(def d : dbcursor){ + results << d + } return results; } @@ -187,14 +193,15 @@ class GaoDeMapCrawl extends BaseBatch{ } def searching(i, location){ - // def bound_gd = location.bound_gd +// def bound_gd = location.bound_gd // def lats = [bound_gd[1], bound_gd[3]] // def lngs = [bound_gd[0], bound_gd[2]] // lats.sort() // lngs.sort() -// println index -// if(lngs[0]<=121.418153 && lngs[1]>=121.418153 && lats[0]<=31.218824 && lats[1]>=31.218824){ - search(i, location) + println i+" "+ location +// 120.019556,30.313701 + //if(lngs[0]<=120.019556 && lngs[1]>=120.019556 && lats[0]<=30.313701 && lats[1]>=30.313701){ + search(i, location) // } if(updateLists.size() > 0 && updateLists.size()>1000){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index 53823ee85..d69846af1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -45,15 +45,18 @@ class Esf58Crawl extends BaseBatch{ void process(Object args) { //addContent("福州","https://fz.58.com/xiaoqu/", 70); - for(def i=0;i<5;i++){ - def city = "福州" - def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); - distMap.put(city, districts) + def citys = CommonDatautils.findCitys() + for(def city : citys){ + for(def i=0;i<1;i++){ +// def city = "福州" + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); + distMap.put(city, districts) - addCrawlKeys(city); - addDetailContent(city, "https://fz.58.com/xiaoqu/"); + addCrawlKeys(city); + addDetailContent(city, "https://fz.58.com/xiaoqu/"); - Thread.sleep(1000*60*10) + Thread.sleep(1000*60*10) + } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index d0ebd044c..a8b9a0427 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -17,7 +17,7 @@ class SynchDistrictPrice { def districtDetailColl = CrawalDBUtil.getDistrictDetailColl(); def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房", "anjuke.com":"安居客","cityhouse.cn":"城市房产"] -// def sourceMap = ["cityhouse.cn":"城市房产"] +// def sourceMap = ["lianjia.com":"链家"] static main(args) { def task = new SynchDistrictPrice() task.run() @@ -43,6 +43,7 @@ class SynchDistrictPrice { def q = new BasicDBObject() q.put("city", city) q.put("root", root) +// q.put("name","兰州万达城") q.put('m_date', new BasicDBObject('$gte', curMonthFirstDay)); def keys = new BasicDBObject() keys.put("city", 1) @@ -95,6 +96,8 @@ class SynchDistrictPrice { thirdPartyColl.remove(new BasicDBObject('_id',obj._id)) addIndex++ thirdPartyColl.save(r) + }else{ + thirdPartyColl.save(r) } // if(!obj){ // addIndex++