Merge remote-tracking branch 'origin/master'
This commit is contained in:
+7
-7
@@ -70,12 +70,12 @@ class BigdataDealBatch extends BaseBatch{
|
||||
//664b73800000000000000000 5-21
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
|
||||
println "curDate:"+curDate
|
||||
def allCitys = CommonDatautils.findAllCity()
|
||||
for(def city : allCitys){
|
||||
addDealPriceMatch(city);
|
||||
}
|
||||
updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
|
||||
// println "curDate:"+curDate
|
||||
// def allCitys = CommonDatautils.findAllCity()
|
||||
// for(def city : allCitys){
|
||||
// addDealPriceMatch(city);
|
||||
// }
|
||||
// updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
|
||||
|
||||
|
||||
def rows = []
|
||||
@@ -385,7 +385,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
def converData(def trade, def cityCodeDatas, def distCodeMap){
|
||||
def ret = []
|
||||
def data = distCodeMap.get(trade.region)
|
||||
def data = distCodeMap.get(trade.match_region)
|
||||
if(!data){
|
||||
println trade.city +" "+trade.region+" 没有找到对应的区域"
|
||||
return ret
|
||||
|
||||
+4
-2
@@ -237,7 +237,7 @@ class CcbCommonUtils {
|
||||
def lists = []
|
||||
println dist.name +" "+total
|
||||
for(def d : dbcursor){
|
||||
def key=d.city+"##"+d.match_region+"##"+d.match_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
|
||||
def key=d.city+"##"+d.match_region+"##"+d.match_name+"##"+d.area+"##"+d.avg_price+"##"+d.total_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
|
||||
if(!rets.contains(key)){
|
||||
rets << key
|
||||
d.dist_id = dist._id.toString()
|
||||
@@ -444,7 +444,7 @@ class CcbCommonUtils {
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13
|
||||
// q.put("name","同心花园")
|
||||
// q.put("name","荣和怡景园")
|
||||
// q.put("name_cat", 0);
|
||||
// q.put("name", new BasicDBObject('$in', ["中信泰富又一城一期","中信泰富又一城"]))
|
||||
|
||||
@@ -734,6 +734,8 @@ class CcbCommonUtils {
|
||||
if(count>0){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("trade_date", endDate)
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("flag", "补漏变更成交时间")
|
||||
DEAL_PRICE_COLL.updateMulti(q, new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
|
||||
+37
-1
@@ -120,6 +120,7 @@ class CommonDatautils {
|
||||
|
||||
public static final DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009")
|
||||
public static final DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
public static final DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com")
|
||||
|
||||
public static DSCollection getHouseDataStoreConnection(){
|
||||
if(house_dataStoreConnection != null) {
|
||||
@@ -4651,6 +4652,8 @@ class CommonDatautils {
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","区域")
|
||||
q.put("district_diff",1)
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("province",1)
|
||||
keys.put("city",1)
|
||||
@@ -4658,6 +4661,7 @@ class CommonDatautils {
|
||||
keys.put("street",1)
|
||||
keys.put("mzb_district_name", 1)
|
||||
keys.put("shape", 1)
|
||||
keys.put("district_short",1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000)
|
||||
def regionMap = [:]
|
||||
for(def d : dbcursor){
|
||||
@@ -4672,6 +4676,8 @@ class CommonDatautils {
|
||||
println d.province+" "+d.district +" 格式化不出民政部区域"
|
||||
}
|
||||
}
|
||||
|
||||
println "区域加载完成"
|
||||
return regionMap
|
||||
}
|
||||
|
||||
@@ -4700,8 +4706,9 @@ class CommonDatautils {
|
||||
list << d
|
||||
townMap.put(key, list)
|
||||
}
|
||||
return townMap
|
||||
|
||||
println "乡镇街道 加载完成"
|
||||
return townMap
|
||||
}
|
||||
|
||||
|
||||
@@ -4760,6 +4767,21 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
static def queryDistNameCityByMapping(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","区域")
|
||||
q.put("district_diff", 1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
|
||||
def retMap = [:]
|
||||
for(def d : dbcursor){
|
||||
if(d.district_short && d.city_short){
|
||||
retMap.put(d.city_short+"#"+d.district_short, d)
|
||||
}
|
||||
}
|
||||
return retMap;
|
||||
}
|
||||
|
||||
|
||||
public static def findAllDistrictByCity(def city, def cat, def fields){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
@@ -4853,5 +4875,19 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
static def saveOptLog(city, region, name, cat, opt, user_name){
|
||||
def obj = new BasicDBObject();
|
||||
obj.put("city",city);
|
||||
obj.put("region",region);
|
||||
obj.put("district_name",name);
|
||||
obj.put("category",cat);
|
||||
obj.put("c_date", new Date())
|
||||
obj.put("m_date", new Date())
|
||||
obj.put("type", opt)
|
||||
obj.put("user_name", user_name)
|
||||
obj.put("origin", "script")
|
||||
OPERATION_LOG_COLL.save(obj)
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user