Merge branch 'master' of http://gitlab.fangjia.com/dev0/main
This commit is contained in:
+2
-2
@@ -11,7 +11,7 @@ class CcbCodeUtils {
|
||||
static def parseBuildYear(String buildDate) {
|
||||
if(buildDate){
|
||||
String years = DataUtil.match(buildDate, Pattern.compile("\\d{4}"))
|
||||
if(years){
|
||||
if(years && years.isNumber() && years.toInteger()>1900 && years.toInteger()<2022){
|
||||
return years.toInteger()
|
||||
}
|
||||
}
|
||||
@@ -159,6 +159,6 @@ class CcbCodeUtils {
|
||||
return 2;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
package com.fangjia.script.gj.ccb
|
||||
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
class FileUtils {
|
||||
static def batchCreateFile(ret, fileName, headers,charset){
|
||||
File file = new File(fileName);
|
||||
if (file.exists()) {
|
||||
file.delete();
|
||||
}
|
||||
|
||||
OutputStream out = new FileOutputStream(fileName);
|
||||
def sb = new StringBuffer();
|
||||
|
||||
sb.append(headers.join("|@|"))
|
||||
sb.append(System.getProperty("line.separator"));
|
||||
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
|
||||
def index = 0
|
||||
for(def row : ret){
|
||||
index++
|
||||
sb.append(StringUtils.join(row.toArray(), "|@|"))
|
||||
sb.append(System.getProperty("line.separator"));
|
||||
println index
|
||||
|
||||
if(index%10000==0){
|
||||
println index+"¿ªÊ¼Ð´ÈëCSV "+new Date().toLocaleString()
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
println index+"½áÊøÐ´ÈëCSV "+new Date().toLocaleString()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(sb.length()>0){
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
}
|
||||
out.close();
|
||||
}
|
||||
}
|
||||
+161
-86
@@ -11,6 +11,7 @@ import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
@@ -51,7 +52,7 @@ class HouseExport {
|
||||
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
static def totalCount = new AtomicLong()
|
||||
|
||||
|
||||
def cityCodeMap = ["北京":"110100","重庆":"500100", "哈尔滨":"230100"]
|
||||
def regionCodeMap = [:]
|
||||
@@ -60,22 +61,37 @@ class HouseExport {
|
||||
def task = new HouseExport();
|
||||
|
||||
task.run()
|
||||
|
||||
}
|
||||
|
||||
def run(){
|
||||
// "重庆",
|
||||
def citys = ["北京"]
|
||||
for(def city : citys){
|
||||
getRegionCodeByCity(city)
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address","alias","finish_building_date","house_property","property_life","build_type","estate_type2","build_structure"]);
|
||||
|
||||
def city = "重庆"
|
||||
getRegionCodeByCity(city)
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias","finish_building_date","house_property","property_life","build_type","estate_type2","build_structure"]);
|
||||
if(city == "哈尔滨"){
|
||||
exportSellCSV(city, districts, 20170801, 20210301, 2020);
|
||||
// exportRentCSV(city, districts, 20170801, 20210301, 2020);
|
||||
|
||||
// exportSellCSV(city, districts, 20200201, 20210301, 2020);
|
||||
// exportRentCSV(city, districts, 20200201, 20210301, 2020);
|
||||
}else{
|
||||
|
||||
exportSellCSV(city, districts);
|
||||
// exportRentCSV(city, districts);
|
||||
exportRentCSV(city, districts, 20170801, 20181001, 2017)
|
||||
exportRentCSV(city, districts, 20200201, 20210301, 2020);
|
||||
|
||||
exportSellCSV(city, districts, 20170801, 20181001, 2017);
|
||||
exportSellCSV(city, districts, 20200201, 20210301, 2020);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def exportSellCSV(city, districts){
|
||||
def exportSellCSV(city, districts, startDate, endDate, year){
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
println city +" 小区量:"+districts.size()
|
||||
def datas = []
|
||||
@@ -83,11 +99,12 @@ class HouseExport {
|
||||
def retMap = new ConcurrentHashMap()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def rMap = districts[i];
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def ret = findHouses(city, "sell", 20170801,20180901, rMap.name, rMap.alias)
|
||||
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias)
|
||||
if(ret){
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def regionCode = regionCodeMap.get(city +"-"+rMap.region)
|
||||
@@ -98,6 +115,7 @@ class HouseExport {
|
||||
def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure);
|
||||
for(def m : ret){
|
||||
m.city_code = cityCode
|
||||
m.region = rMap.region
|
||||
m.region_code = regionCode
|
||||
m.district_id = rMap.get("_id").toString()
|
||||
m.district_name = rMap.name
|
||||
@@ -108,6 +126,7 @@ class HouseExport {
|
||||
m.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall);
|
||||
m.direction_code = CcbCodeUtils.getDirectionCode(m.direction)
|
||||
m.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment);
|
||||
m.address = rMap.address
|
||||
m.build_structure_code = struCode
|
||||
m.house_structure_code = null//getHouseStructureCode();
|
||||
m.source = "fjw"
|
||||
@@ -124,19 +143,24 @@ class HouseExport {
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
println " 去重后:"+retMap.values()
|
||||
def allDatas = createSellDatas(retMap.values());
|
||||
def allDatas = createSellDatas(retMap);
|
||||
def headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司"]
|
||||
batchCreateFile(allDatas, "d:\\数据\\建行\\bigdata_forsale_${cityCodeMap[city]}.csv",headers, "utf-8");
|
||||
println city +" [sell]生成完成!"
|
||||
|
||||
def count = 1;
|
||||
for(def allData : allDatas){
|
||||
FileUtils.batchCreateFile(allData, "d:\\数据\\建行\\bigdata_forsale_${year}_${cityCodeMap[city]}_${count}.dat",headers, "utf-8");
|
||||
println city +" [sell-${count}]生成完成!"
|
||||
count ++;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
def exportRentCSV(city, districts){
|
||||
def exportRentCSV(city, districts, startDate, endDate, year){
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
println city +" 小区量:"+districts.size()
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
|
||||
def retMap = new ConcurrentHashMap()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
@@ -145,7 +169,7 @@ class HouseExport {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def ret = findHouses(city, "rent", 20170801,20180901, rMap.name, rMap.alias)
|
||||
def ret = findHouses(city, rMap.region,"rent", startDate,endDate, rMap.name, rMap.alias)
|
||||
if(ret){
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def regionCode = regionCodeMap.get(city +"-"+rMap.region)
|
||||
@@ -157,8 +181,10 @@ class HouseExport {
|
||||
for(def m : ret){
|
||||
m.city_code = cityCode
|
||||
m.region_code = regionCode
|
||||
m.region = rMap.region
|
||||
m.district_id = rMap.get("_id").toString()
|
||||
m.district_name = rMap.name
|
||||
m.address = rMap.address
|
||||
m.build_year = buildYear;
|
||||
m.proptype = housePro
|
||||
m.propyears = propertyYear;
|
||||
@@ -184,20 +210,24 @@ class HouseExport {
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
println " 去重后:"+retMap.values()
|
||||
def allDatas = createRentDatas(retMap.values());
|
||||
def headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","出租面积","单价","总价","租赁方式","出租付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司"]
|
||||
batchCreateFile(allDatas, "d:\\数据\\建行\\bigdata_forrent_${cityCodeMap[city]}.csv", headers, "utf-8");
|
||||
println city +" [rent]生成完成!"
|
||||
|
||||
def allDatas = createRentDatas(retMap);
|
||||
def headers = ["城市ID","出租ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","出租面积","单价","总价","租赁方式","出租付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司"]
|
||||
|
||||
|
||||
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\bigdata_forrent_${year}_${cityCodeMap[city]}.dat", headers, "utf-8");
|
||||
println city +" [rent]生成完成!"
|
||||
}
|
||||
|
||||
def createSellDatas(def list){
|
||||
def createSellDatas(def map){
|
||||
def datas1 = []
|
||||
def datas2 = []
|
||||
def datas = []
|
||||
for(def m : list){
|
||||
map.each{k,m->
|
||||
def data = []
|
||||
data << m.city_code
|
||||
data << m.houseId
|
||||
//data << m.houseId
|
||||
data << new ObjectId()
|
||||
data << ""
|
||||
data << m.city
|
||||
data << m.region_code
|
||||
@@ -248,18 +278,30 @@ class HouseExport {
|
||||
data << m.m_date==null?"":m.m_date
|
||||
data << m.source
|
||||
|
||||
datas << data
|
||||
if(datas1.size()>2000000){
|
||||
datas2 << data
|
||||
}else{
|
||||
datas1 << data
|
||||
}
|
||||
}
|
||||
if(datas1.size()>0){
|
||||
datas << datas1
|
||||
}
|
||||
|
||||
if(datas2.size()>0){
|
||||
datas << datas2
|
||||
}
|
||||
return datas ;
|
||||
}
|
||||
|
||||
|
||||
def createRentDatas(def list){
|
||||
def createRentDatas(def map){
|
||||
def datas = []
|
||||
for(def m : list){
|
||||
map.each{k,m->
|
||||
def data = []
|
||||
data << m.city_code
|
||||
data << m.houseId
|
||||
// data << m.houseId
|
||||
data << new ObjectId()
|
||||
data << ""
|
||||
data << m.city
|
||||
data << m.region_code
|
||||
@@ -278,6 +320,8 @@ class HouseExport {
|
||||
data << m.proptype==null?"":m.proptype
|
||||
data << m.propyears==null?"":m.propyears
|
||||
data << m.area==null?"":m.area
|
||||
|
||||
|
||||
data << m.avg_price==null?"":m.avg_price
|
||||
data << m.total_price==null?"":m.total_price
|
||||
data << m.rent_type_code==null?"":m.rent_type_code
|
||||
@@ -287,7 +331,12 @@ class HouseExport {
|
||||
data << m.estate_type==null?"":m.estate_type
|
||||
data << m.house_structure_code==null?"":m.house_structure_code
|
||||
data << m.build_structure_code==null?"":m.build_structure_code
|
||||
data << m.area==null?"":m.area
|
||||
if(m.rent_type_code && m.rent_type_code==1){
|
||||
data << ""
|
||||
}else{
|
||||
data << m.area==null?"":m.area
|
||||
}
|
||||
|
||||
data << ""
|
||||
data << m.house_type==null?"":m.house_type
|
||||
data << m.room==null?"":m.room
|
||||
@@ -318,43 +367,7 @@ class HouseExport {
|
||||
}
|
||||
|
||||
|
||||
def batchCreateFile(ret, fileName, headers,charset){
|
||||
File file = new File(fileName);
|
||||
if (file.exists()) {
|
||||
file.delete();
|
||||
}
|
||||
|
||||
OutputStream out = new FileOutputStream(fileName);
|
||||
def sb = new StringBuffer();
|
||||
|
||||
sb.append(headers.join("@|@"))
|
||||
sb.append(System.getProperty("line.separator"));
|
||||
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
|
||||
def index = 0
|
||||
for(def row : ret){
|
||||
index++
|
||||
sb.append(StringUtils.join(row.toArray(), "|@|"))
|
||||
sb.append(System.getProperty("line.separator"));
|
||||
println index
|
||||
|
||||
if(index%10000==0){
|
||||
println index+"开始写入CSV "+new Date().toLocaleString()
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
println index+"结束写入CSV "+new Date().toLocaleString()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(sb.length()>0){
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
}
|
||||
out.close();
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -490,12 +503,12 @@ class HouseExport {
|
||||
}
|
||||
|
||||
|
||||
def findHouses(def city, def cat, def startDate, def endDate, def name, def alias){
|
||||
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", "sell"))
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(date_range).must(area_range)
|
||||
|
||||
if(!alias){
|
||||
@@ -521,6 +534,13 @@ class HouseExport {
|
||||
def map = houses[k];
|
||||
DBObject seawater = new BasicDBObject(map)
|
||||
|
||||
if(!seawater.total_price){
|
||||
continue;
|
||||
}
|
||||
if(!seawater.avg_price){
|
||||
continue;
|
||||
}
|
||||
|
||||
if(!seawater.height){
|
||||
seawater.height = null
|
||||
}
|
||||
@@ -534,8 +554,14 @@ class HouseExport {
|
||||
// if(!seawater.room){
|
||||
// continue;
|
||||
// }
|
||||
if(seawater.room && (seawater.room>=5 || seawater.room<1)){
|
||||
continue;
|
||||
if(!seawater.room){
|
||||
seawater.room = null
|
||||
}
|
||||
|
||||
if(seawater.room){
|
||||
if(seawater.room>=5 || seawater.room<1){
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if(!seawater.hall){
|
||||
@@ -551,28 +577,48 @@ class HouseExport {
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
def floor = parseFloor(seawater.floor_info)
|
||||
if(seawater.height && floor && floor.isNumber()){
|
||||
if(floor.toInteger()>seawater.height){
|
||||
floor = null
|
||||
seawater.height = null
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(!floor){
|
||||
seawater.floor = null
|
||||
}
|
||||
if(floor && floor.toInteger()<1){
|
||||
floor = null
|
||||
}
|
||||
|
||||
def rowMap = [:];
|
||||
if(cat == "rent"){
|
||||
rowMap.rent_type = seawater.rent_type
|
||||
if(seawater.avg_price && seawater.avg_price>1000){
|
||||
continue
|
||||
}
|
||||
|
||||
if(seawater.avg_price && seawater.avg_price<10){
|
||||
continue
|
||||
}
|
||||
|
||||
}
|
||||
rowMap.houseId = seawater.id
|
||||
rowMap.city = city
|
||||
rowMap.region =seawater.region
|
||||
rowMap.address = seawater.address
|
||||
//rowMap.region =seawater.region
|
||||
//rowMap.address = seawater.address
|
||||
|
||||
|
||||
rowMap.area = seawater.area
|
||||
rowMap.total_price = seawater.total_price
|
||||
rowMap.avg_price = seawater.avg_price
|
||||
rowMap.height = seawater.height=="null"?null:seawater.height
|
||||
if(rowMap.height && rowMap.height<1){
|
||||
rowMap.height = null
|
||||
}
|
||||
rowMap.floor = floor=="null"?null:floor
|
||||
rowMap.room = seawater.room=="null"?null:seawater.room
|
||||
rowMap.hall = seawater.hall=="null"?null:seawater.hall
|
||||
@@ -584,15 +630,36 @@ class HouseExport {
|
||||
def fitment = parseFitment(seawater.fitment)
|
||||
rowMap.fitment = fitment=="null"?null:fitment
|
||||
|
||||
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
|
||||
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
|
||||
rowMap.m_date = DateUtils.getConverDate(seawater.m_date)
|
||||
|
||||
|
||||
if(seawater.p_date instanceof String){
|
||||
seawater.p_date = seawater.p_date.toInteger()
|
||||
}
|
||||
if(city=="哈尔滨" && seawater.p_date>=20181001 && seawater.p_date<20200201){
|
||||
rowMap.p_date = DateUtils.getRandomDate("2018-01-01", "2018-10-01")
|
||||
rowMap.c_date = rowMap.p_date
|
||||
rowMap.m_date = rowMap.p_date
|
||||
}else{
|
||||
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
|
||||
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
|
||||
rowMap.m_date = DateUtils.getConverDate(seawater.m_date)
|
||||
}
|
||||
|
||||
rets << rowMap
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def getPublicDate(def pDate){
|
||||
|
||||
if(pDate>=20181001 && pDate<20200201){
|
||||
return DateUtils.getRandomDate("2018-10-01", "2020-02-01")
|
||||
}
|
||||
def pdate = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(pDate), "yyyy-MM-dd HH:mm:ss")
|
||||
return pdate;
|
||||
}
|
||||
|
||||
static def parseFloor(def floorDesc){
|
||||
if(floorDesc){
|
||||
floorDesc = floorDesc.replaceAll("[\\s ]+", "")
|
||||
@@ -607,18 +674,18 @@ class HouseExport {
|
||||
if (matcher1.find()) {
|
||||
return matcher1.group(1)
|
||||
}
|
||||
if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){
|
||||
return "中层"
|
||||
}
|
||||
|
||||
if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){
|
||||
return "高层"
|
||||
}
|
||||
|
||||
|
||||
if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){
|
||||
return "低层"
|
||||
}
|
||||
// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){
|
||||
// return "中层"
|
||||
// }
|
||||
//
|
||||
// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){
|
||||
// return "高层"
|
||||
// }
|
||||
//
|
||||
//
|
||||
// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){
|
||||
// return "低层"
|
||||
// }
|
||||
if(floorDesc.isNumber()){
|
||||
return floorDesc;
|
||||
}
|
||||
@@ -700,6 +767,14 @@ class HouseExport {
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
* 生成一个用不重复的ID
|
||||
*/
|
||||
def GenNonDuplicateID(randomLength){
|
||||
return Number(Math.random().toString().substr(3,randomLength) + Date.now()).toString(36)
|
||||
}
|
||||
//GenNonDuplicateID()将生成 rfmipbs8ag0kgkcogc 类似的ID
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
+199
@@ -0,0 +1,199 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class AnjukeCrawlToDB {
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
|
||||
static main(args) {
|
||||
def task = new AnjukeCrawlToDB();
|
||||
|
||||
// task.run()
|
||||
|
||||
task.stat()
|
||||
}
|
||||
|
||||
def stat(){
|
||||
def districts = CommonDatautils.findAllDistricts("北京", "district", ["estate_charge","finish_building_date","build_area","household_count","car_site_count","cubage_percent","virescence_percent","estate_company","developer"]);
|
||||
|
||||
def total = districts.size()
|
||||
def estate_chargeCount = 0
|
||||
def build_areaCount = 0
|
||||
def household_countCount = 0
|
||||
def finish_building_dateCount = 0
|
||||
def car_site_countCount = 0
|
||||
def cubage_percentCount = 0
|
||||
def virescence_percentCount = 0
|
||||
def developerCount = 0
|
||||
def estate_companyCount = 0
|
||||
for(def dists : districts){
|
||||
if(dists.estate_charge){
|
||||
estate_chargeCount++
|
||||
}
|
||||
// 总建面积
|
||||
if(dists.build_area){
|
||||
build_areaCount++
|
||||
}
|
||||
// 总数
|
||||
if(dists.household_count){
|
||||
household_countCount++
|
||||
}
|
||||
// 建造年代
|
||||
if(dists.finish_building_date){
|
||||
finish_building_dateCount++
|
||||
}
|
||||
// 停车位
|
||||
if(dists.car_site_count){
|
||||
car_site_countCount++
|
||||
}
|
||||
// 容积率
|
||||
if(dists.cubage_percent){
|
||||
cubage_percentCount++
|
||||
}
|
||||
// 绿化率
|
||||
if(dists.virescence_percent){
|
||||
virescence_percentCount++
|
||||
}
|
||||
// 开发商
|
||||
if(dists.developer){
|
||||
developerCount++
|
||||
}
|
||||
// 物业公司
|
||||
if(dists.estate_company){
|
||||
estate_companyCount++
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def estate_charge_pre = DataHandleUtils.getDoubleRound(2, estate_chargeCount*100/total)
|
||||
def build_area_pre = DataHandleUtils.getDoubleRound(2, build_areaCount*100/total)
|
||||
def household_count_pre = DataHandleUtils.getDoubleRound(2, household_countCount*100/total)
|
||||
def finish_building_date_pre = DataHandleUtils.getDoubleRound(2, finish_building_dateCount*100/total)
|
||||
def car_site_pre = DataHandleUtils.getDoubleRound(2, car_site_countCount*100/total)
|
||||
def cubage_percent_pre = DataHandleUtils.getDoubleRound(2, cubage_percentCount*100/total)
|
||||
def virescence_percent_pre = DataHandleUtils.getDoubleRound(2, virescence_percentCount*100/total)
|
||||
def developer_pre = DataHandleUtils.getDoubleRound(2, developerCount*100/total)
|
||||
def estate_company_pre = DataHandleUtils.getDoubleRound(2, estate_companyCount*100/total)
|
||||
|
||||
println finish_building_date_pre
|
||||
println virescence_percent_pre
|
||||
println cubage_percent_pre
|
||||
|
||||
println build_area_pre
|
||||
println estate_charge_pre
|
||||
println developer_pre
|
||||
println estate_company_pre
|
||||
// println household_count_pre
|
||||
// println car_site_pre
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def run(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\待抓取小区_安居客_detail.xlsx",0,1)
|
||||
|
||||
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
for(def i=0;i<datas.size();i++){
|
||||
def data = datas[i]
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
totalCount.addAndGet(1)
|
||||
findData(data);
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def findData(data){
|
||||
def dists = CommonDatautils.findDistrictById(data['A'])
|
||||
if(dists){
|
||||
def udata = new BasicDBObject()
|
||||
//物业费
|
||||
if(!dists.estate_charge && data['K']){
|
||||
def estateCharge = data['K'].replace("元/平米/月","")
|
||||
if(estateCharge && estateCharge.toDouble()>0){
|
||||
udata.put("estate_charge",estateCharge)
|
||||
}
|
||||
}
|
||||
// 总建面积
|
||||
if(!dists.build_area && data['L']){
|
||||
def buildArea = data['L'].toString().replace("m?","")
|
||||
if(buildArea){
|
||||
udata.put("build_area",buildArea.toDouble().toInteger().toString())
|
||||
}
|
||||
|
||||
}
|
||||
// 总数
|
||||
if(!dists.household_count && data['M']){
|
||||
def householdCount= data['M']
|
||||
if(householdCount && householdCount.toString().isNumber()){
|
||||
udata.put("household_count",householdCount.toInteger())
|
||||
}
|
||||
}
|
||||
// 建造年代
|
||||
if(!dists.finish_building_date && data['N']){
|
||||
def year = data['N'].replace("年","")
|
||||
if(year && year.isNumber()){
|
||||
udata.put("finish_building_date",year.toString())
|
||||
}
|
||||
}
|
||||
// 停车位
|
||||
if(!dists.car_site_count && data['O']){
|
||||
if(data['O'].isNumber()){
|
||||
udata.put("car_site_count", data['O'].toInteger())
|
||||
}
|
||||
}
|
||||
// 容积率
|
||||
if(!dists.cubage_percent && data['R']){
|
||||
def cubage_percent = data['R']
|
||||
if(cubage_percent && cubage_percent.isNumber() && cubage_percent.toDouble()<10){
|
||||
udata.put("cubage_percent", cubage_percent.toString())
|
||||
}
|
||||
}
|
||||
// 绿化率
|
||||
if(!dists.virescence_percent && data['Q']){
|
||||
def virescence_percent = data['Q'].toString().replace("%","")
|
||||
if(virescence_percent && virescence_percent.isNumber()){
|
||||
udata.put("virescence_percent", virescence_percent.toString())
|
||||
}
|
||||
}
|
||||
// 开发商
|
||||
if(!dists.developer && data['R'] && !data['R'].contains("未知") && !data['R'].contains("无")){
|
||||
udata.put("developer", data['R'])
|
||||
}
|
||||
// 物业公司
|
||||
if(!dists.estate_company && data['S'] && !data['S'].contains("未知") && !data['S'].contains("无")){
|
||||
udata.put("estate_company",data['S'])
|
||||
}
|
||||
|
||||
if(udata){
|
||||
udata.put("m_date", new Date())
|
||||
println totalCount.get()+" "+data['A']+" "+udata
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(data['A'])),new BasicDBObject('$set', udata));
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -767,15 +767,19 @@ class DateUtils {
|
||||
}
|
||||
|
||||
static String getConverDate(def mills){
|
||||
String s = null
|
||||
Date date = null;
|
||||
if(mills instanceof String){
|
||||
s = mills.replace("T"," ").replace(".000Z","")
|
||||
date = com.fangjia.core.util.DateUtil.parseISODate(mills)
|
||||
Calendar cal = Calendar.getInstance();
|
||||
cal.setTime(date);
|
||||
cal.add(Calendar.HOUR, 8);// 24小时制
|
||||
date = cal.getTime();
|
||||
//s = mills.replace("T"," ").replace(".000Z","")
|
||||
}else if(mills instanceof Long){
|
||||
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
||||
Date date = new Date(mills);
|
||||
s = dateFormat.format(date);
|
||||
date = new Date(mills)
|
||||
}
|
||||
|
||||
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
||||
String s = dateFormat.format(date);
|
||||
return s
|
||||
}
|
||||
|
||||
@@ -914,6 +918,13 @@ class DateUtils {
|
||||
SimpleDateFormat format = new SimpleDateFormat(pattern);
|
||||
return format.parse(s)
|
||||
}
|
||||
|
||||
public static String getRandomDate(def startDate, def endDate){
|
||||
Date randomDate = randomDate(startDate, endDate);
|
||||
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
|
||||
String result = format.format(randomDate);
|
||||
return result;
|
||||
}
|
||||
|
||||
static main(args){
|
||||
|
||||
@@ -939,7 +950,48 @@ class DateUtils {
|
||||
|
||||
//monthDiffCount()
|
||||
|
||||
println between(parseFromTightYYYYMMDD('2009-11-10'), parseFromTightYYYYMMDD('2008-12-12'), Calendar.MONTH)
|
||||
// println between(parseFromTightYYYYMMDD('2009-11-10'), parseFromTightYYYYMMDD('2008-12-12'), Calendar.MONTH)
|
||||
|
||||
// Date randomDate = randomDate("2010-09-20", "2011-09-22");
|
||||
// SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
|
||||
// String result = format.format(randomDate);
|
||||
System.out.println(getRandomDate("2010-09-20", "2011-09-22"));
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 生成随机时间
|
||||
*
|
||||
* @param beginDate
|
||||
* @param endDate
|
||||
* @return
|
||||
*/
|
||||
public static Date randomDate(String beginDate, String endDate) {
|
||||
try {
|
||||
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
|
||||
Date start = format.parse(beginDate);// 构造开始日期
|
||||
Date end = format.parse(endDate);// 构造结束日期
|
||||
// getTime()表示返回自 1970 年 1 月 1 日 00:00:00 GMT 以来此 Date 对象表示的毫秒数。
|
||||
if (start.getTime() >= end.getTime()) {
|
||||
return null;
|
||||
}
|
||||
long date = random(start.getTime(), end.getTime());
|
||||
return new Date(date);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public static long random(long begin, long end) {
|
||||
long rtn = begin + (long) (Math.random() * (end - begin));
|
||||
// 如果返回的是开始时间和结束时间,则递归调用本函数查找随机值
|
||||
if (rtn == begin || rtn == end) {
|
||||
return random(begin, end);
|
||||
}
|
||||
return rtn;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user