Merge remote-tracking branch 'origin/master'

This commit is contained in:
majintao
2024-07-22 13:29:50 +08:00
3 changed files with 48 additions and 10 deletions
@@ -70,12 +70,12 @@ class BigdataDealBatch extends BaseBatch{
//664b73800000000000000000 5-21
def citys = CcbCommonUtils.getRunningCitys(args)
println "curDate:"+curDate
def allCitys = CommonDatautils.findAllCity()
for(def city : allCitys){
addDealPriceMatch(city);
}
updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
// println "curDate:"+curDate
// def allCitys = CommonDatautils.findAllCity()
// for(def city : allCitys){
// addDealPriceMatch(city);
// }
// updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
def rows = []
@@ -385,7 +385,7 @@ class BigdataDealBatch extends BaseBatch{
def converData(def trade, def cityCodeDatas, def distCodeMap){
def ret = []
def data = distCodeMap.get(trade.region)
def data = distCodeMap.get(trade.match_region)
if(!data){
println trade.city +" "+trade.region+" 没有找到对应的区域"
return ret
@@ -237,7 +237,7 @@ class CcbCommonUtils {
def lists = []
println dist.name +" "+total
for(def d : dbcursor){
def key=d.city+"##"+d.match_region+"##"+d.match_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
def key=d.city+"##"+d.match_region+"##"+d.match_name+"##"+d.area+"##"+d.avg_price+"##"+d.total_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
if(!rets.contains(key)){
rets << key
d.dist_id = dist._id.toString()
@@ -444,7 +444,7 @@ class CcbCommonUtils {
q.put("cat", cat);
q.put("city", city);
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13
// q.put("name","同心花园")
// q.put("name","荣和怡景园")
// q.put("name_cat", 0);
// q.put("name", new BasicDBObject('$in', ["中信泰富又一城一期","中信泰富又一城"]))
@@ -734,6 +734,8 @@ class CcbCommonUtils {
if(count>0){
def udata = new BasicDBObject()
udata.put("trade_date", endDate)
udata.put("m_date", new Date())
udata.put("flag", "补漏变更成交时间")
DEAL_PRICE_COLL.updateMulti(q, new BasicDBObject('$set',udata))
}
}
@@ -120,6 +120,7 @@ class CommonDatautils {
public static final DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009")
public static final DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
public static final DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com")
public static DSCollection getHouseDataStoreConnection(){
if(house_dataStoreConnection != null) {
@@ -4651,6 +4652,8 @@ class CommonDatautils {
def q = new BasicDBObject()
q.put("type","区域")
q.put("district_diff",1)
def keys = new BasicDBObject()
keys.put("province",1)
keys.put("city",1)
@@ -4658,6 +4661,7 @@ class CommonDatautils {
keys.put("street",1)
keys.put("mzb_district_name", 1)
keys.put("shape", 1)
keys.put("district_short",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000)
def regionMap = [:]
for(def d : dbcursor){
@@ -4672,6 +4676,8 @@ class CommonDatautils {
println d.province+" "+d.district +" 格式化不出民政部区域"
}
}
println "区域加载完成"
return regionMap
}
@@ -4700,8 +4706,9 @@ class CommonDatautils {
list << d
townMap.put(key, list)
}
return townMap
println "乡镇街道 加载完成"
return townMap
}
@@ -4760,6 +4767,21 @@ class CommonDatautils {
}
static def queryDistNameCityByMapping(){
def q = new BasicDBObject()
q.put("type","区域")
q.put("district_diff", 1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
def retMap = [:]
for(def d : dbcursor){
if(d.district_short && d.city_short){
retMap.put(d.city_short+"#"+d.district_short, d)
}
}
return retMap;
}
public static def findAllDistrictByCity(def city, def cat, def fields){
DBObject q = new BasicDBObject();
@@ -4853,5 +4875,19 @@ class CommonDatautils {
}
static def saveOptLog(city, region, name, cat, opt, user_name){
def obj = new BasicDBObject();
obj.put("city",city);
obj.put("region",region);
obj.put("district_name",name);
obj.put("category",cat);
obj.put("c_date", new Date())
obj.put("m_date", new Date())
obj.put("type", opt)
obj.put("user_name", user_name)
obj.put("origin", "script")
OPERATION_LOG_COLL.save(obj)
}
}