This commit is contained in:
2021-09-23 10:07:54 +08:00
parent b369686290
commit a3ec34c62a
5 changed files with 88 additions and 15 deletions
@@ -103,6 +103,13 @@ class CcbCommonUtils {
alias << dist.name
}
if(dist.name.endsWith("弄小区")){
def name = dist.name.replace("弄小区","弄")
if(!alias.contains(name)){
alias << name
}
}
if(alias){
q.put("district_name", new BasicDBObject('$in', alias))
}
@@ -131,6 +138,37 @@ class CcbCommonUtils {
return lists
}
public static def findTradesDetailByAddress(dist, startDate, endDate, startPrice, endPrice){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
q.put("area", new BasicDBObject('$gte', 20))
q.put("district_name", dist.address)
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
def dbcursor = DEAL_PRICE_COLL.find(q)
def total = dbcursor.count()
def rets = new HashSet()
def lists = []
for(def d : dbcursor){
if(startPrice && d.avg_price>=startPrice && d.avg_price<=endPrice){
def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
if(!rets.contains(key)){
rets << key
d.dist_id = dist._id.toString()
d.address = dist.address
d.master_name = dist.name
d.build_type = dist.build_type
d.build_structure = dist.build_structure
d.house_roperty = dist.house_roperty
d.property_life = dist.property_life
d.estate_type2 = dist.estate_type2
lists << d
}
}
}
return lists
}
public static def findTrades(dist, startDate, endDate){
def q = new BasicDBObject()
@@ -187,7 +225,7 @@ class CcbCommonUtils {
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
// q.put("name", "莱兴公寓")
// q.put("name", "周家湾")
// q.put("name_cat", 0);
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
@@ -1231,6 +1231,28 @@ class CommonDatautils {
return ret;
}
public static def findDistrictByAddress2(def city, def region, def address, def cat){
DBObject q = new BasicDBObject();
if(region){
q.put('region', region)
}
q.put('city',city)
q.put('cat', cat)
q.put('visible',0)
q.put("address", address);
def dsCursor = getDataStoreConnection().find(q);
def ret = [];
for(def obj : dsCursor){
ret << obj;
}
return ret;
}
public static def getCityRegions(city){
@@ -119,10 +119,11 @@ class GaoDeMapCrawl extends BaseBatch{
def run(){//,"西安","重庆"
//def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
def citys =["上海","南京","苏州","无锡","扬州","常州","徐州","镇江","南通","泰州","连云港","盐城","淮安","宿迁","杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"]
def citys = ["上海","南京","苏州","无锡","扬州","常州","徐州","镇江","南通","泰州","连云港","盐城","淮安","宿迁","杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"]
def allcitys = CommonUtils.findAllCity()
def totalCount = 0;
for(def city : allcitys){
if(!citys.contains(city)){
def stop = false
while(stop == false){
@@ -135,6 +136,7 @@ class GaoDeMapCrawl extends BaseBatch{
}
}
}
}
}
@@ -146,7 +148,11 @@ class GaoDeMapCrawl extends BaseBatch{
q.put(prefix+"status", new BasicDBObject(QueryOperators.EXISTS, false))
// q.put("status", "失败")
// q.put("traffic_status", new BasicDBObject(QueryOperators.EXISTS, false))
def results = BOUNDS_COLL.find(q).limit(2000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def dbcursor = BOUNDS_COLL.find(q).batchSize(3000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def results = []
for(def d : dbcursor){
results << d
}
return results;
}
@@ -187,14 +193,15 @@ class GaoDeMapCrawl extends BaseBatch{
}
def searching(i, location){
// def bound_gd = location.bound_gd
// def bound_gd = location.bound_gd
// def lats = [bound_gd[1], bound_gd[3]]
// def lngs = [bound_gd[0], bound_gd[2]]
// lats.sort()
// lngs.sort()
// println index
// if(lngs[0]<=121.418153 && lngs[1]>=121.418153 && lats[0]<=31.218824 && lats[1]>=31.218824){
search(i, location)
println i+" "+ location
// 120.019556,30.313701
//if(lngs[0]<=120.019556 && lngs[1]>=120.019556 && lats[0]<=30.313701 && lats[1]>=30.313701){
search(i, location)
// }
if(updateLists.size() > 0 && updateLists.size()>1000){
@@ -45,15 +45,18 @@ class Esf58Crawl extends BaseBatch{
void process(Object args) {
//addContent("福州","https://fz.58.com/xiaoqu/", 70);
for(def i=0;i<5;i++){
def city = "福州"
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
distMap.put(city, districts)
def citys = CommonDatautils.findCitys()
for(def city : citys){
for(def i=0;i<1;i++){
// def city = "福州"
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
distMap.put(city, districts)
addCrawlKeys(city);
addDetailContent(city, "https://fz.58.com/xiaoqu/");
addCrawlKeys(city);
addDetailContent(city, "https://fz.58.com/xiaoqu/");
Thread.sleep(1000*60*10)
Thread.sleep(1000*60*10)
}
}
}
@@ -17,7 +17,7 @@ class SynchDistrictPrice {
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
"anjuke.com":"安居客","cityhouse.cn":"城市房产"]
// def sourceMap = ["cityhouse.cn":"城市房产"]
// def sourceMap = ["lianjia.com":"链家"]
static main(args) {
def task = new SynchDistrictPrice()
task.run()
@@ -43,6 +43,7 @@ class SynchDistrictPrice {
def q = new BasicDBObject()
q.put("city", city)
q.put("root", root)
// q.put("name","兰州万达城")
q.put('m_date', new BasicDBObject('$gte', curMonthFirstDay));
def keys = new BasicDBObject()
keys.put("city", 1)
@@ -95,6 +96,8 @@ class SynchDistrictPrice {
thirdPartyColl.remove(new BasicDBObject('_id',obj._id))
addIndex++
thirdPartyColl.save(r)
}else{
thirdPartyColl.save(r)
}
// if(!obj){
// addIndex++