This commit is contained in:
2021-10-13 12:46:29 +08:00
parent c19c765a51
commit e64fa84ab8
3 changed files with 11 additions and 12 deletions
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20210930);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20200101, 20210930);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
}
}
@@ -96,9 +96,9 @@ class BigdataForsaleBatch extends BaseBatch{
def commId = rMap.get("_id").toString()
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias)
// if(ret && ret.size()<100){
// ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias)
// }
if(ret && ret.size()<100){
ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias)
}
if(ret){
def areaData = distCodeMap.get(rMap.region)
if(areaData){
@@ -159,8 +159,6 @@ class BigdataForsaleBatch extends BaseBatch{
}
FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true);
println "txt文件生成 "
}
@@ -124,9 +124,9 @@ class CcbCommonUtils {
def lists = []
for(def d : dbcursor){
if(startPrice && d.avg_price>=startPrice && d.avg_price<=endPrice){
def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
if(!rets.contains(key)){
rets << key
//def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
//if(!rets.contains(key)){
// rets << key
d.dist_id = dist._id.toString()
d.address = dist.address
d.master_name = dist.name
@@ -136,7 +136,7 @@ class CcbCommonUtils {
d.property_life = dist.property_life
d.estate_type2 = dist.estate_type2
lists << d
}
//}
}
}
return lists
@@ -229,7 +229,7 @@ class CcbCommonUtils {
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
// q.put("name", "周家湾")
// q.put("name", "海昌欣城")
// q.put("name_cat", 0);
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
@@ -19,7 +19,7 @@ import java.util.concurrent.Executors
@BatchOwner(Watcher.GaoJuan)
class ThirdDataToGatherBatch extends BaseBatch{
def static final SHORT_DATE = 202107//Utils.getLastMonth()
def static final SHORT_DATE = Utils.getLastMonth()
def static final DBCollection dataProcessColl = CrawalDBUtil.getDataProcessCol();
//MongoUtils.getMongoDBByAuth().getCollection("data_process")
def static final DBCollection AVERAGE_STATISTICS_OFFLINE = CrawalDBUtil.getAverage_statistics_col();
@@ -41,6 +41,7 @@ class ThirdDataToGatherBatch extends BaseBatch{
@Override
void process(Object args) {
println SHORT_DATE
def citys = CommonDatautils.findAllCity();
def weekend = DateUtils.getFirstDayOfWeek(new Date());
//def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightYYYYMMDD("2021-07-15"));