diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy index e3cb5bfee..c20405c02 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy @@ -11,7 +11,7 @@ class CcbCodeUtils { static def parseBuildYear(String buildDate) { if(buildDate){ String years = DataUtil.match(buildDate, Pattern.compile("\\d{4}")) - if(years){ + if(years && years.isNumber() && years.toInteger()>1900){ return years.toInteger() } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy index 7027cf373..3dbeebc2f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy @@ -11,6 +11,7 @@ import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject import com.mongodb.DBObject import org.apache.commons.lang.StringUtils +import org.bson.types.ObjectId import org.elasticsearch.index.query.BoolQueryBuilder import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders @@ -51,7 +52,7 @@ class HouseExport { public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) - static def totalCount = new AtomicLong() + def cityCodeMap = ["北京":"110100","重庆":"500100", "哈尔滨":"230100"] def regionCodeMap = [:] @@ -60,24 +61,25 @@ class HouseExport { def task = new HouseExport(); task.run() - } def run(){ - def city = "北京" - getRegionCodeByCity(city) - def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias","finish_building_date","house_property","property_life","build_type","estate_type2","build_structure"]); - - - exportSellCSV(city, districts); -// exportRentCSV(city, districts); + def citys = ["北京", "哈尔滨", "重庆"] + for(def city : citys){ + getRegionCodeByCity(city) + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias","finish_building_date","house_property","property_life","build_type","estate_type2","build_structure"]); + exportSellCSV(city, districts, 20170801, 20180901, 2017); + exportRentCSV(city, districts, 20170801, 20180901, 2017); + exportSellCSV(city, districts, 20200201, 20210201, 2020); + exportRentCSV(city, districts, 20200201, 20210201, 2020); + } } - def exportSellCSV(city, districts){ - + def exportSellCSV(city, districts, startDate, endDate, year){ + def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(districts.size()) println city +" 小区量:"+districts.size() def datas = [] @@ -89,7 +91,7 @@ class HouseExport { void run() { try { totalCount.addAndGet(1) - def ret = findHouses(city, "sell", 20170801,20180901, rMap.name, rMap.alias) + def ret = findHouses(city, "sell", startDate,endDate, rMap.name, rMap.alias) if(ret){ def cityCode = cityCodeMap.get(city) def regionCode = regionCodeMap.get(city +"-"+rMap.region) @@ -131,7 +133,7 @@ class HouseExport { def count = 1; for(def allData : allDatas){ - FileUtils.batchCreateFile(allData, "d:\\数据\\建行\\bigdata_forsale_2017_${cityCodeMap[city]}_${count}.csv",headers, "utf-8"); + FileUtils.batchCreateFile(allData, "d:\\数据\\建行\\bigdata_forsale_${year}_${cityCodeMap[city]}_${count}.csv",headers, "utf-8"); println city +" [sell-${count}]生成完成!" count ++; } @@ -140,10 +142,10 @@ class HouseExport { } - def exportRentCSV(city, districts){ + def exportRentCSV(city, districts, startDate, endDate, year){ final CountDownLatch latch = new CountDownLatch(districts.size()) println city +" 小区量:"+districts.size() - + def totalCount = new AtomicLong() def retMap = new ConcurrentHashMap() for(def i=0;i def data = [] data << m.city_code - data << m.houseId + //data << m.houseId + data << new ObjectId() data << "" data << m.city data << m.region_code @@ -279,7 +283,8 @@ class HouseExport { map.each{k,m-> def data = [] data << m.city_code - data << m.houseId +// data << m.houseId + data << new ObjectId() data << "" data << m.city data << m.region_code @@ -479,7 +484,7 @@ class HouseExport { QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false); QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); - QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", "sell")) + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", cat)) .must(date_range).must(area_range) if(!alias){ @@ -541,11 +546,17 @@ class HouseExport { floor = null seawater.height = null } + if(floor && floor.toInteger()<1){ + floor = null + } } def rowMap = [:]; if(cat == "rent"){ rowMap.rent_type = seawater.rent_type + if(seawater.avg_price && seawater.avg_price>1000){ + continue + } } rowMap.houseId = seawater.id rowMap.city = city @@ -557,6 +568,9 @@ class HouseExport { rowMap.total_price = seawater.total_price rowMap.avg_price = seawater.avg_price rowMap.height = seawater.height=="null"?null:seawater.height + if(rowMap.height && rowMap.height<1){ + rowMap.height = null + } rowMap.floor = floor=="null"?null:floor rowMap.room = seawater.room=="null"?null:seawater.room rowMap.hall = seawater.hall=="null"?null:seawater.hall @@ -591,18 +605,18 @@ class HouseExport { if (matcher1.find()) { return matcher1.group(1) } - if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){ - return "中层" - } - - if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){ - return "高层" - } - - - if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){ - return "低层" - } +// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){ +// return "中层" +// } +// +// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){ +// return "高层" +// } +// +// +// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){ +// return "低层" +// } if(floorDesc.isNumber()){ return floorDesc; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy index 8bc5d9198..8a2c8c385 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy @@ -767,15 +767,19 @@ class DateUtils { } static String getConverDate(def mills){ - String s = null + Date date = null; if(mills instanceof String){ - s = mills.replace("T"," ").replace(".000Z","") + date = com.fangjia.core.util.DateUtil.parseISODate(mills) + Calendar cal = Calendar.getInstance(); + cal.setTime(date); + cal.add(Calendar.HOUR, 8);// 24小时制 + date = cal.getTime(); + //s = mills.replace("T"," ").replace(".000Z","") }else if(mills instanceof Long){ - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - Date date = new Date(mills); - s = dateFormat.format(date); + date = new Date(mills) } - + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + String s = dateFormat.format(date); return s }