+39
-1
@@ -103,6 +103,13 @@ class CcbCommonUtils {
|
||||
alias << dist.name
|
||||
}
|
||||
|
||||
if(dist.name.endsWith("弄小区")){
|
||||
def name = dist.name.replace("弄小区","弄")
|
||||
if(!alias.contains(name)){
|
||||
alias << name
|
||||
}
|
||||
}
|
||||
|
||||
if(alias){
|
||||
q.put("district_name", new BasicDBObject('$in', alias))
|
||||
}
|
||||
@@ -131,6 +138,37 @@ class CcbCommonUtils {
|
||||
return lists
|
||||
}
|
||||
|
||||
public static def findTradesDetailByAddress(dist, startDate, endDate, startPrice, endPrice){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
q.put("area", new BasicDBObject('$gte', 20))
|
||||
q.put("district_name", dist.address)
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def total = dbcursor.count()
|
||||
def rets = new HashSet()
|
||||
def lists = []
|
||||
for(def d : dbcursor){
|
||||
if(startPrice && d.avg_price>=startPrice && d.avg_price<=endPrice){
|
||||
def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
|
||||
if(!rets.contains(key)){
|
||||
rets << key
|
||||
d.dist_id = dist._id.toString()
|
||||
d.address = dist.address
|
||||
d.master_name = dist.name
|
||||
d.build_type = dist.build_type
|
||||
d.build_structure = dist.build_structure
|
||||
d.house_roperty = dist.house_roperty
|
||||
d.property_life = dist.property_life
|
||||
d.estate_type2 = dist.estate_type2
|
||||
lists << d
|
||||
}
|
||||
}
|
||||
}
|
||||
return lists
|
||||
}
|
||||
|
||||
|
||||
public static def findTrades(dist, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
@@ -187,7 +225,7 @@ class CcbCommonUtils {
|
||||
q.put("visible",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name", "莱兴公寓")
|
||||
// q.put("name", "周家湾")
|
||||
// q.put("name_cat", 0);
|
||||
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
|
||||
|
||||
|
||||
+22
@@ -1231,6 +1231,28 @@ class CommonDatautils {
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
public static def findDistrictByAddress2(def city, def region, def address, def cat){
|
||||
DBObject q = new BasicDBObject();
|
||||
if(region){
|
||||
q.put('region', region)
|
||||
}
|
||||
|
||||
q.put('city',city)
|
||||
q.put('cat', cat)
|
||||
q.put('visible',0)
|
||||
q.put("address", address);
|
||||
|
||||
|
||||
def dsCursor = getDataStoreConnection().find(q);
|
||||
def ret = [];
|
||||
for(def obj : dsCursor){
|
||||
ret << obj;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
public static def getCityRegions(city){
|
||||
|
||||
@@ -119,10 +119,11 @@ class GaoDeMapCrawl extends BaseBatch{
|
||||
|
||||
def run(){//,"西安","重庆"
|
||||
//def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
def citys =["上海","南京","苏州","无锡","扬州","常州","徐州","镇江","南通","泰州","连云港","盐城","淮安","宿迁","杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"]
|
||||
def citys = ["上海","南京","苏州","无锡","扬州","常州","徐州","镇江","南通","泰州","连云港","盐城","淮安","宿迁","杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"]
|
||||
def allcitys = CommonUtils.findAllCity()
|
||||
def totalCount = 0;
|
||||
for(def city : allcitys){
|
||||
|
||||
if(!citys.contains(city)){
|
||||
def stop = false
|
||||
while(stop == false){
|
||||
@@ -135,6 +136,7 @@ class GaoDeMapCrawl extends BaseBatch{
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -146,7 +148,11 @@ class GaoDeMapCrawl extends BaseBatch{
|
||||
q.put(prefix+"status", new BasicDBObject(QueryOperators.EXISTS, false))
|
||||
// q.put("status", "失败")
|
||||
// q.put("traffic_status", new BasicDBObject(QueryOperators.EXISTS, false))
|
||||
def results = BOUNDS_COLL.find(q).limit(2000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def dbcursor = BOUNDS_COLL.find(q).batchSize(3000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def results = []
|
||||
for(def d : dbcursor){
|
||||
results << d
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
@@ -187,14 +193,15 @@ class GaoDeMapCrawl extends BaseBatch{
|
||||
}
|
||||
|
||||
def searching(i, location){
|
||||
// def bound_gd = location.bound_gd
|
||||
// def bound_gd = location.bound_gd
|
||||
// def lats = [bound_gd[1], bound_gd[3]]
|
||||
// def lngs = [bound_gd[0], bound_gd[2]]
|
||||
// lats.sort()
|
||||
// lngs.sort()
|
||||
// println index
|
||||
// if(lngs[0]<=121.418153 && lngs[1]>=121.418153 && lats[0]<=31.218824 && lats[1]>=31.218824){
|
||||
search(i, location)
|
||||
println i+" "+ location
|
||||
// 120.019556,30.313701
|
||||
//if(lngs[0]<=120.019556 && lngs[1]>=120.019556 && lats[0]<=30.313701 && lats[1]>=30.313701){
|
||||
search(i, location)
|
||||
// }
|
||||
|
||||
if(updateLists.size() > 0 && updateLists.size()>1000){
|
||||
|
||||
+10
-7
@@ -45,15 +45,18 @@ class Esf58Crawl extends BaseBatch{
|
||||
void process(Object args) {
|
||||
//addContent("福州","https://fz.58.com/xiaoqu/", 70);
|
||||
|
||||
for(def i=0;i<5;i++){
|
||||
def city = "福州"
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
|
||||
distMap.put(city, districts)
|
||||
def citys = CommonDatautils.findCitys()
|
||||
for(def city : citys){
|
||||
for(def i=0;i<1;i++){
|
||||
// def city = "福州"
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
|
||||
distMap.put(city, districts)
|
||||
|
||||
addCrawlKeys(city);
|
||||
addDetailContent(city, "https://fz.58.com/xiaoqu/");
|
||||
addCrawlKeys(city);
|
||||
addDetailContent(city, "https://fz.58.com/xiaoqu/");
|
||||
|
||||
Thread.sleep(1000*60*10)
|
||||
Thread.sleep(1000*60*10)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+4
-1
@@ -17,7 +17,7 @@ class SynchDistrictPrice {
|
||||
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
|
||||
def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
|
||||
"anjuke.com":"安居客","cityhouse.cn":"城市房产"]
|
||||
// def sourceMap = ["cityhouse.cn":"城市房产"]
|
||||
// def sourceMap = ["lianjia.com":"链家"]
|
||||
static main(args) {
|
||||
def task = new SynchDistrictPrice()
|
||||
task.run()
|
||||
@@ -43,6 +43,7 @@ class SynchDistrictPrice {
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("root", root)
|
||||
// q.put("name","兰州万达城")
|
||||
q.put('m_date', new BasicDBObject('$gte', curMonthFirstDay));
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city", 1)
|
||||
@@ -95,6 +96,8 @@ class SynchDistrictPrice {
|
||||
thirdPartyColl.remove(new BasicDBObject('_id',obj._id))
|
||||
addIndex++
|
||||
thirdPartyColl.save(r)
|
||||
}else{
|
||||
thirdPartyColl.save(r)
|
||||
}
|
||||
// if(!obj){
|
||||
// addIndex++
|
||||
|
||||
Reference in New Issue
Block a user