Merge branch 'script_gj' into 'master'

update

See merge request !649
This commit is contained in:
2023-05-26 18:38:45 +08:00
4 changed files with 55 additions and 3 deletions
@@ -218,6 +218,7 @@ class CcbCommonUtils {
q.put("avg_price", new BasicDBObject('$gt', 0))
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
q.put("match_id", dist._id.toString())
q.put("type", new BasicDBObject('$ne','车位'))
def dbcursor = DEAL_PRICE_COLL.find(q)
def total = dbcursor.count()
def rets = new HashSet()
@@ -445,12 +446,17 @@ class CcbCommonUtils {
def dbsour = DBUtils.seaweed.find(q,keys).batchSize(2000)
println "[${city}] 原始:"+dbsour.count()
def rets = []
def ids = []
for(def d : dbsour){
if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2) && !d.name.contains("洋房") && !d.name.contains("公房")){
if(d.address){
d.address = d.address.replaceAll("\r|\n|\\s", "");
}
rets << d
if(!ids.contains(d._id.toString())){
ids << d._id.toString()
rets << d
}
}
}
println "[${city}] 小区量:"+rets.size()
@@ -606,11 +612,12 @@ class CcbCommonUtils {
public static def findTradesByMonth(city, startDate, endDate){
public static def findTradesByMonth(city, startDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
q.put("trade_date", new BasicDBObject('$gte', startDate))
q.put("match_id", new BasicDBObject('$exists', false))
// q.put("district_name","国贸天琴湾")
def dbcursor = DEAL_PRICE_COLL.find(q)
def total = dbcursor.count()
def rets = new HashSet()
@@ -71,6 +71,7 @@ class CommonDatautils {
public static final DBCollection LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia_base", "loudong", mongoDbUrl)
public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", mongoDbUrl)
public static final DBCollection CITY_INFO_COLL = MongoUtils.getCollectionByName2("fangjia_data_ana", "city_info", mongoDbUrl)
public static final DBCollection REGION_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "region_info", MongoUtils.MongoDB_Domain_mongo);
public static DSCollection getHouseDataStoreConnection(){
if(house_dataStoreConnection != null) {
@@ -2989,6 +2990,19 @@ class CommonDatautils {
}
return names;
}
public static def getCityByRegion(region){
DBObject q = new BasicDBObject();
q.put("category", "region");
q.put("name", region);
def cursor = REGION_BLOCK_COLL.find(q);
def citys = [];
for(def obj : cursor){
citys << obj.city;
}
return citys;
}
static def findBusLineCount(def city, def maxLat, def minLat, def maxLng, def minLng){
DBObject q = new BasicDBObject()
@@ -3464,5 +3478,17 @@ class CommonDatautils {
}
return null
}
static def findRegionPriceRate(city, region){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
def cur = REGION_INFO_CONN.find(q).sort(new BasicDBObject("weekend", -1)).limit(1)
def obj = null
while(cur.hasNext()) {
obj = cur.next()
}
return obj
}
}
@@ -59,6 +59,7 @@ class CZDataBatch extends BaseBatch{
def districts = converList(CommonDatautils.findAllDistricts(city, "district", fields));
districtRun(districts, statMap, commids);
FzDataUtils.removeNoUpdate(city, districts)
def shops = CommonDatautils.findAllDistricts(city, "shop", fields);
shops = mergeShop(shops);
@@ -101,6 +101,23 @@ class FzDataUtils {
saveDistrict(map);
}
static def removeNoUpdate(city, districts){
def ids = []
for(def d : districts){
ids << d._id.toString()
}
DBObject q = new BasicDBObject();
q.put("city", city)
q.put("cat", "district")
def dbcursor = DIST_COLL.find(q)
for(def cur : dbcursor){
if(!ids.contains(cur.num_id)){
def del_data = new BasicDBObject()
del_data.put("num_id", cur.num_id)
DIST_COLL.remove(del_data)
}
}
}
static def saveDistrict(def m){
DBObject q = new BasicDBObject();
@@ -111,6 +128,7 @@ class FzDataUtils {
m.each {k,v->
udata.put(k, v);
}
udata.put("m_date", new Date())
if(dbo){
udata.removeField("num_id")
DIST_COLL.update(q, new BasicDBObject('$set', udata), false, true);