This commit is contained in:
2021-10-13 12:46:29 +08:00
parent c19c765a51
commit e64fa84ab8
3 changed files with 11 additions and 12 deletions
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
// def distIds = getDistrictIds(cityCodeDataMap.get(city)) // def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20210930); exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20200101, 20210930);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
} }
} }
@@ -96,9 +96,9 @@ class BigdataForsaleBatch extends BaseBatch{
def commId = rMap.get("_id").toString() def commId = rMap.get("_id").toString()
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias) def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias)
// if(ret && ret.size()<100){ if(ret && ret.size()<100){
// ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias) ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias)
// } }
if(ret){ if(ret){
def areaData = distCodeMap.get(rMap.region) def areaData = distCodeMap.get(rMap.region)
if(areaData){ if(areaData){
@@ -159,8 +159,6 @@ class BigdataForsaleBatch extends BaseBatch{
} }
FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true); FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true);
println "txt文件生成 " println "txt文件生成 "
} }
@@ -124,9 +124,9 @@ class CcbCommonUtils {
def lists = [] def lists = []
for(def d : dbcursor){ for(def d : dbcursor){
if(startPrice && d.avg_price>=startPrice && d.avg_price<=endPrice){ if(startPrice && d.avg_price>=startPrice && d.avg_price<=endPrice){
def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd") //def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
if(!rets.contains(key)){ //if(!rets.contains(key)){
rets << key // rets << key
d.dist_id = dist._id.toString() d.dist_id = dist._id.toString()
d.address = dist.address d.address = dist.address
d.master_name = dist.name d.master_name = dist.name
@@ -136,7 +136,7 @@ class CcbCommonUtils {
d.property_life = dist.property_life d.property_life = dist.property_life
d.estate_type2 = dist.estate_type2 d.estate_type2 = dist.estate_type2
lists << d lists << d
} //}
} }
} }
return lists return lists
@@ -229,7 +229,7 @@ class CcbCommonUtils {
q.put("visible",0) q.put("visible",0)
q.put("cat", cat); q.put("cat", cat);
q.put("city", city); q.put("city", city);
// q.put("name", "周家湾") // q.put("name", "海昌欣城")
// q.put("name_cat", 0); // q.put("name_cat", 0);
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
@@ -19,7 +19,7 @@ import java.util.concurrent.Executors
@BatchOwner(Watcher.GaoJuan) @BatchOwner(Watcher.GaoJuan)
class ThirdDataToGatherBatch extends BaseBatch{ class ThirdDataToGatherBatch extends BaseBatch{
def static final SHORT_DATE = 202107//Utils.getLastMonth() def static final SHORT_DATE = Utils.getLastMonth()
def static final DBCollection dataProcessColl = CrawalDBUtil.getDataProcessCol(); def static final DBCollection dataProcessColl = CrawalDBUtil.getDataProcessCol();
//MongoUtils.getMongoDBByAuth().getCollection("data_process") //MongoUtils.getMongoDBByAuth().getCollection("data_process")
def static final DBCollection AVERAGE_STATISTICS_OFFLINE = CrawalDBUtil.getAverage_statistics_col(); def static final DBCollection AVERAGE_STATISTICS_OFFLINE = CrawalDBUtil.getAverage_statistics_col();
@@ -41,6 +41,7 @@ class ThirdDataToGatherBatch extends BaseBatch{
@Override @Override
void process(Object args) { void process(Object args) {
println SHORT_DATE
def citys = CommonDatautils.findAllCity(); def citys = CommonDatautils.findAllCity();
def weekend = DateUtils.getFirstDayOfWeek(new Date()); def weekend = DateUtils.getFirstDayOfWeek(new Date());
//def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightYYYYMMDD("2021-07-15")); //def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightYYYYMMDD("2021-07-15"));