This commit is contained in:
2021-03-15 00:16:59 +08:00
parent 1d4318142b
commit 7be22202e3
3 changed files with 59 additions and 41 deletions
@@ -11,7 +11,7 @@ class CcbCodeUtils {
static def parseBuildYear(String buildDate) {
if(buildDate){
String years = DataUtil.match(buildDate, Pattern.compile("\\d{4}"))
if(years){
if(years && years.isNumber() && years.toInteger()>1900){
return years.toInteger()
}
}
@@ -11,6 +11,7 @@ import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import org.apache.commons.lang.StringUtils
import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
@@ -51,7 +52,7 @@ class HouseExport {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
static def totalCount = new AtomicLong()
def cityCodeMap = ["北京":"110100","重庆":"500100", "哈尔滨":"230100"]
def regionCodeMap = [:]
@@ -60,24 +61,25 @@ class HouseExport {
def task = new HouseExport();
task.run()
}
def run(){
def city = "北京"
getRegionCodeByCity(city)
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias","finish_building_date","house_property","property_life","build_type","estate_type2","build_structure"]);
exportSellCSV(city, districts);
// exportRentCSV(city, districts);
def citys = ["北京", "哈尔滨", "重庆"]
for(def city : citys){
getRegionCodeByCity(city)
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias","finish_building_date","house_property","property_life","build_type","estate_type2","build_structure"]);
exportSellCSV(city, districts, 20170801, 20180901, 2017);
exportRentCSV(city, districts, 20170801, 20180901, 2017);
exportSellCSV(city, districts, 20200201, 20210201, 2020);
exportRentCSV(city, districts, 20200201, 20210201, 2020);
}
}
def exportSellCSV(city, districts){
def exportSellCSV(city, districts, startDate, endDate, year){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
println city +" 小区量:"+districts.size()
def datas = []
@@ -89,7 +91,7 @@ class HouseExport {
void run() {
try {
totalCount.addAndGet(1)
def ret = findHouses(city, "sell", 20170801,20180901, rMap.name, rMap.alias)
def ret = findHouses(city, "sell", startDate,endDate, rMap.name, rMap.alias)
if(ret){
def cityCode = cityCodeMap.get(city)
def regionCode = regionCodeMap.get(city +"-"+rMap.region)
@@ -131,7 +133,7 @@ class HouseExport {
def count = 1;
for(def allData : allDatas){
FileUtils.batchCreateFile(allData, "d:\\数据\\建行\\bigdata_forsale_2017_${cityCodeMap[city]}_${count}.csv",headers, "utf-8");
FileUtils.batchCreateFile(allData, "d:\\数据\\建行\\bigdata_forsale_${year}_${cityCodeMap[city]}_${count}.csv",headers, "utf-8");
println city +" [sell-${count}]生成完成!"
count ++;
}
@@ -140,10 +142,10 @@ class HouseExport {
}
def exportRentCSV(city, districts){
def exportRentCSV(city, districts, startDate, endDate, year){
final CountDownLatch latch = new CountDownLatch(districts.size())
println city +" 小区量:"+districts.size()
def totalCount = new AtomicLong()
def retMap = new ConcurrentHashMap()
for(def i=0;i<districts.size();i++){
@@ -152,7 +154,7 @@ class HouseExport {
void run() {
try {
totalCount.addAndGet(1)
def ret = findHouses(city, "rent", 20170801,20180901, rMap.name, rMap.alias)
def ret = findHouses(city, "rent", startDate,endDate, rMap.name, rMap.alias)
if(ret){
def cityCode = cityCodeMap.get(city)
def regionCode = regionCodeMap.get(city +"-"+rMap.region)
@@ -193,10 +195,11 @@ class HouseExport {
latch.await()
def allDatas = createRentDatas(retMap);
def headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","出租面积","单价","总价","租赁方式","出租付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\bigdata_forrent_${cityCodeMap[city]}.csv", headers, "utf-8");
println city +" [rent]生成完成!"
def headers = ["城市ID","出租ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","出租面积","单价","总价","租赁方式","出租付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\bigdata_forrent_${year}_${cityCodeMap[city]}.csv", headers, "utf-8");
println city +" [rent]生成完成!"
}
def createSellDatas(def map){
@@ -206,7 +209,8 @@ class HouseExport {
map.each{k,m->
def data = []
data << m.city_code
data << m.houseId
//data << m.houseId
data << new ObjectId()
data << ""
data << m.city
data << m.region_code
@@ -279,7 +283,8 @@ class HouseExport {
map.each{k,m->
def data = []
data << m.city_code
data << m.houseId
// data << m.houseId
data << new ObjectId()
data << ""
data << m.city
data << m.region_code
@@ -479,7 +484,7 @@ class HouseExport {
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", "sell"))
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range)
if(!alias){
@@ -541,11 +546,17 @@ class HouseExport {
floor = null
seawater.height = null
}
if(floor && floor.toInteger()<1){
floor = null
}
}
def rowMap = [:];
if(cat == "rent"){
rowMap.rent_type = seawater.rent_type
if(seawater.avg_price && seawater.avg_price>1000){
continue
}
}
rowMap.houseId = seawater.id
rowMap.city = city
@@ -557,6 +568,9 @@ class HouseExport {
rowMap.total_price = seawater.total_price
rowMap.avg_price = seawater.avg_price
rowMap.height = seawater.height=="null"?null:seawater.height
if(rowMap.height && rowMap.height<1){
rowMap.height = null
}
rowMap.floor = floor=="null"?null:floor
rowMap.room = seawater.room=="null"?null:seawater.room
rowMap.hall = seawater.hall=="null"?null:seawater.hall
@@ -591,18 +605,18 @@ class HouseExport {
if (matcher1.find()) {
return matcher1.group(1)
}
if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){
return "中层"
}
if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){
return "高层"
}
if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){
return "低层"
}
// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){
// return "中层"
// }
//
// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){
// return "高层"
// }
//
//
// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){
// return "低层"
// }
if(floorDesc.isNumber()){
return floorDesc;
}
@@ -767,15 +767,19 @@ class DateUtils {
}
static String getConverDate(def mills){
String s = null
Date date = null;
if(mills instanceof String){
s = mills.replace("T"," ").replace(".000Z","")
date = com.fangjia.core.util.DateUtil.parseISODate(mills)
Calendar cal = Calendar.getInstance();
cal.setTime(date);
cal.add(Calendar.HOUR, 8);// 24小时制
date = cal.getTime();
//s = mills.replace("T"," ").replace(".000Z","")
}else if(mills instanceof Long){
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
Date date = new Date(mills);
s = dateFormat.format(date);
date = new Date(mills)
}
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String s = dateFormat.format(date);
return s
}