update
This commit is contained in:
+1
-1
@@ -11,7 +11,7 @@ class CcbCodeUtils {
|
||||
static def parseBuildYear(String buildDate) {
|
||||
if(buildDate){
|
||||
String years = DataUtil.match(buildDate, Pattern.compile("\\d{4}"))
|
||||
if(years){
|
||||
if(years && years.isNumber() && years.toInteger()>1900){
|
||||
return years.toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
+48
-34
@@ -11,6 +11,7 @@ import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
@@ -51,7 +52,7 @@ class HouseExport {
|
||||
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
static def totalCount = new AtomicLong()
|
||||
|
||||
|
||||
def cityCodeMap = ["北京":"110100","重庆":"500100", "哈尔滨":"230100"]
|
||||
def regionCodeMap = [:]
|
||||
@@ -60,24 +61,25 @@ class HouseExport {
|
||||
def task = new HouseExport();
|
||||
|
||||
task.run()
|
||||
|
||||
}
|
||||
|
||||
def run(){
|
||||
|
||||
def city = "北京"
|
||||
getRegionCodeByCity(city)
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias","finish_building_date","house_property","property_life","build_type","estate_type2","build_structure"]);
|
||||
|
||||
|
||||
exportSellCSV(city, districts);
|
||||
// exportRentCSV(city, districts);
|
||||
def citys = ["北京", "哈尔滨", "重庆"]
|
||||
for(def city : citys){
|
||||
getRegionCodeByCity(city)
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias","finish_building_date","house_property","property_life","build_type","estate_type2","build_structure"]);
|
||||
exportSellCSV(city, districts, 20170801, 20180901, 2017);
|
||||
exportRentCSV(city, districts, 20170801, 20180901, 2017);
|
||||
|
||||
exportSellCSV(city, districts, 20200201, 20210201, 2020);
|
||||
exportRentCSV(city, districts, 20200201, 20210201, 2020);
|
||||
}
|
||||
}
|
||||
|
||||
def exportSellCSV(city, districts){
|
||||
|
||||
def exportSellCSV(city, districts, startDate, endDate, year){
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
println city +" 小区量:"+districts.size()
|
||||
def datas = []
|
||||
@@ -89,7 +91,7 @@ class HouseExport {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def ret = findHouses(city, "sell", 20170801,20180901, rMap.name, rMap.alias)
|
||||
def ret = findHouses(city, "sell", startDate,endDate, rMap.name, rMap.alias)
|
||||
if(ret){
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def regionCode = regionCodeMap.get(city +"-"+rMap.region)
|
||||
@@ -131,7 +133,7 @@ class HouseExport {
|
||||
|
||||
def count = 1;
|
||||
for(def allData : allDatas){
|
||||
FileUtils.batchCreateFile(allData, "d:\\数据\\建行\\bigdata_forsale_2017_${cityCodeMap[city]}_${count}.csv",headers, "utf-8");
|
||||
FileUtils.batchCreateFile(allData, "d:\\数据\\建行\\bigdata_forsale_${year}_${cityCodeMap[city]}_${count}.csv",headers, "utf-8");
|
||||
println city +" [sell-${count}]生成完成!"
|
||||
count ++;
|
||||
}
|
||||
@@ -140,10 +142,10 @@ class HouseExport {
|
||||
}
|
||||
|
||||
|
||||
def exportRentCSV(city, districts){
|
||||
def exportRentCSV(city, districts, startDate, endDate, year){
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
println city +" 小区量:"+districts.size()
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
|
||||
def retMap = new ConcurrentHashMap()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
@@ -152,7 +154,7 @@ class HouseExport {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def ret = findHouses(city, "rent", 20170801,20180901, rMap.name, rMap.alias)
|
||||
def ret = findHouses(city, "rent", startDate,endDate, rMap.name, rMap.alias)
|
||||
if(ret){
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def regionCode = regionCodeMap.get(city +"-"+rMap.region)
|
||||
@@ -193,10 +195,11 @@ class HouseExport {
|
||||
latch.await()
|
||||
|
||||
def allDatas = createRentDatas(retMap);
|
||||
def headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","出租面积","单价","总价","租赁方式","出租付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司"]
|
||||
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\bigdata_forrent_${cityCodeMap[city]}.csv", headers, "utf-8");
|
||||
println city +" [rent]生成完成!"
|
||||
def headers = ["城市ID","出租ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","出租面积","单价","总价","租赁方式","出租付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司"]
|
||||
|
||||
|
||||
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\bigdata_forrent_${year}_${cityCodeMap[city]}.csv", headers, "utf-8");
|
||||
println city +" [rent]生成完成!"
|
||||
}
|
||||
|
||||
def createSellDatas(def map){
|
||||
@@ -206,7 +209,8 @@ class HouseExport {
|
||||
map.each{k,m->
|
||||
def data = []
|
||||
data << m.city_code
|
||||
data << m.houseId
|
||||
//data << m.houseId
|
||||
data << new ObjectId()
|
||||
data << ""
|
||||
data << m.city
|
||||
data << m.region_code
|
||||
@@ -279,7 +283,8 @@ class HouseExport {
|
||||
map.each{k,m->
|
||||
def data = []
|
||||
data << m.city_code
|
||||
data << m.houseId
|
||||
// data << m.houseId
|
||||
data << new ObjectId()
|
||||
data << ""
|
||||
data << m.city
|
||||
data << m.region_code
|
||||
@@ -479,7 +484,7 @@ class HouseExport {
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", "sell"))
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(date_range).must(area_range)
|
||||
|
||||
if(!alias){
|
||||
@@ -541,11 +546,17 @@ class HouseExport {
|
||||
floor = null
|
||||
seawater.height = null
|
||||
}
|
||||
if(floor && floor.toInteger()<1){
|
||||
floor = null
|
||||
}
|
||||
}
|
||||
|
||||
def rowMap = [:];
|
||||
if(cat == "rent"){
|
||||
rowMap.rent_type = seawater.rent_type
|
||||
if(seawater.avg_price && seawater.avg_price>1000){
|
||||
continue
|
||||
}
|
||||
}
|
||||
rowMap.houseId = seawater.id
|
||||
rowMap.city = city
|
||||
@@ -557,6 +568,9 @@ class HouseExport {
|
||||
rowMap.total_price = seawater.total_price
|
||||
rowMap.avg_price = seawater.avg_price
|
||||
rowMap.height = seawater.height=="null"?null:seawater.height
|
||||
if(rowMap.height && rowMap.height<1){
|
||||
rowMap.height = null
|
||||
}
|
||||
rowMap.floor = floor=="null"?null:floor
|
||||
rowMap.room = seawater.room=="null"?null:seawater.room
|
||||
rowMap.hall = seawater.hall=="null"?null:seawater.hall
|
||||
@@ -591,18 +605,18 @@ class HouseExport {
|
||||
if (matcher1.find()) {
|
||||
return matcher1.group(1)
|
||||
}
|
||||
if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){
|
||||
return "中层"
|
||||
}
|
||||
|
||||
if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){
|
||||
return "高层"
|
||||
}
|
||||
|
||||
|
||||
if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){
|
||||
return "低层"
|
||||
}
|
||||
// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){
|
||||
// return "中层"
|
||||
// }
|
||||
//
|
||||
// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){
|
||||
// return "高层"
|
||||
// }
|
||||
//
|
||||
//
|
||||
// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){
|
||||
// return "低层"
|
||||
// }
|
||||
if(floorDesc.isNumber()){
|
||||
return floorDesc;
|
||||
}
|
||||
|
||||
@@ -767,15 +767,19 @@ class DateUtils {
|
||||
}
|
||||
|
||||
static String getConverDate(def mills){
|
||||
String s = null
|
||||
Date date = null;
|
||||
if(mills instanceof String){
|
||||
s = mills.replace("T"," ").replace(".000Z","")
|
||||
date = com.fangjia.core.util.DateUtil.parseISODate(mills)
|
||||
Calendar cal = Calendar.getInstance();
|
||||
cal.setTime(date);
|
||||
cal.add(Calendar.HOUR, 8);// 24小时制
|
||||
date = cal.getTime();
|
||||
//s = mills.replace("T"," ").replace(".000Z","")
|
||||
}else if(mills instanceof Long){
|
||||
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
||||
Date date = new Date(mills);
|
||||
s = dateFormat.format(date);
|
||||
date = new Date(mills)
|
||||
}
|
||||
|
||||
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
||||
String s = dateFormat.format(date);
|
||||
return s
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user