From e64fa84ab808f73518bbbcab1865c1bc14d63bc7 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 13 Oct 2021 12:46:29 +0800 Subject: [PATCH] update --- .../script/ccbevaluation/BigdataForsaleBatch.groovy | 10 ++++------ .../fangjia/script/ccbevaluation/CcbCommonUtils.groovy | 10 +++++----- .../thirdPriceCrawl/ThirdDataToGatherBatch.groovy | 3 ++- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index db708a6ca..ab705a2a5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20210930); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20200101, 20210930); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); } } @@ -96,9 +96,9 @@ class BigdataForsaleBatch extends BaseBatch{ def commId = rMap.get("_id").toString() def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias) -// if(ret && ret.size()<100){ -// ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias) -// } + if(ret && ret.size()<100){ + ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias) + } if(ret){ def areaData = distCodeMap.get(rMap.region) if(areaData){ @@ -159,8 +159,6 @@ class BigdataForsaleBatch extends BaseBatch{ } FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true); println "txt文件生成 " - - } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index eb080803a..2124be80f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -124,9 +124,9 @@ class CcbCommonUtils { def lists = [] for(def d : dbcursor){ if(startPrice && d.avg_price>=startPrice && d.avg_price<=endPrice){ - def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd") - if(!rets.contains(key)){ - rets << key + //def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd") + //if(!rets.contains(key)){ +// rets << key d.dist_id = dist._id.toString() d.address = dist.address d.master_name = dist.name @@ -136,7 +136,7 @@ class CcbCommonUtils { d.property_life = dist.property_life d.estate_type2 = dist.estate_type2 lists << d - } + //} } } return lists @@ -229,7 +229,7 @@ class CcbCommonUtils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name", "周家湾") +// q.put("name", "海昌欣城") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy index 3f598a077..ddc282cb5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy @@ -19,7 +19,7 @@ import java.util.concurrent.Executors @BatchOwner(Watcher.GaoJuan) class ThirdDataToGatherBatch extends BaseBatch{ - def static final SHORT_DATE = 202107//Utils.getLastMonth() + def static final SHORT_DATE = Utils.getLastMonth() def static final DBCollection dataProcessColl = CrawalDBUtil.getDataProcessCol(); //MongoUtils.getMongoDBByAuth().getCollection("data_process") def static final DBCollection AVERAGE_STATISTICS_OFFLINE = CrawalDBUtil.getAverage_statistics_col(); @@ -41,6 +41,7 @@ class ThirdDataToGatherBatch extends BaseBatch{ @Override void process(Object args) { + println SHORT_DATE def citys = CommonDatautils.findAllCity(); def weekend = DateUtils.getFirstDayOfWeek(new Date()); //def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightYYYYMMDD("2021-07-15"));