diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy index e3cb5bfee..89dff40d4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy @@ -11,7 +11,7 @@ class CcbCodeUtils { static def parseBuildYear(String buildDate) { if(buildDate){ String years = DataUtil.match(buildDate, Pattern.compile("\\d{4}")) - if(years){ + if(years && years.isNumber() && years.toInteger()>1900 && years.toInteger()<2022){ return years.toInteger() } } @@ -159,6 +159,6 @@ class CcbCodeUtils { return 2; } } - return null; + return 0; } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy new file mode 100644 index 000000000..09f80b9da --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy @@ -0,0 +1,43 @@ +package com.fangjia.script.gj.ccb + +import org.apache.commons.lang.StringUtils + +class FileUtils { + static def batchCreateFile(ret, fileName, headers,charset){ + File file = new File(fileName); + if (file.exists()) { + file.delete(); + } + + OutputStream out = new FileOutputStream(fileName); + def sb = new StringBuffer(); + + sb.append(headers.join("|@|")) + sb.append(System.getProperty("line.separator")); + + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + + def index = 0 + for(def row : ret){ + index++ + sb.append(StringUtils.join(row.toArray(), "|@|")) + sb.append(System.getProperty("line.separator")); + println index + + if(index%10000==0){ + println index+"开始写入CSV "+new Date().toLocaleString() + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + println index+"结束写入CSV "+new Date().toLocaleString() + } + + } + + if(sb.length()>0){ + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + } + out.close(); + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy index a3bad5266..3c4ca8b31 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy @@ -11,6 +11,7 @@ import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject import com.mongodb.DBObject import org.apache.commons.lang.StringUtils +import org.bson.types.ObjectId import org.elasticsearch.index.query.BoolQueryBuilder import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders @@ -51,7 +52,7 @@ class HouseExport { public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) - static def totalCount = new AtomicLong() + def cityCodeMap = ["北京":"110100","重庆":"500100", "哈尔滨":"230100"] def regionCodeMap = [:] @@ -60,22 +61,37 @@ class HouseExport { def task = new HouseExport(); task.run() - } def run(){ +// "重庆", + def citys = ["北京"] + for(def city : citys){ + getRegionCodeByCity(city) + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address","alias","finish_building_date","house_property","property_life","build_type","estate_type2","build_structure"]); - def city = "重庆" - getRegionCodeByCity(city) - def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias","finish_building_date","house_property","property_life","build_type","estate_type2","build_structure"]); + if(city == "哈尔滨"){ + exportSellCSV(city, districts, 20170801, 20210301, 2020); +// exportRentCSV(city, districts, 20170801, 20210301, 2020); +// exportSellCSV(city, districts, 20200201, 20210301, 2020); +// exportRentCSV(city, districts, 20200201, 20210301, 2020); + }else{ - exportSellCSV(city, districts); -// exportRentCSV(city, districts); + exportRentCSV(city, districts, 20170801, 20181001, 2017) + exportRentCSV(city, districts, 20200201, 20210301, 2020); + exportSellCSV(city, districts, 20170801, 20181001, 2017); + exportSellCSV(city, districts, 20200201, 20210301, 2020); + + } + + } } - def exportSellCSV(city, districts){ + def exportSellCSV(city, districts, startDate, endDate, year){ + + def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(districts.size()) println city +" 小区量:"+districts.size() def datas = [] @@ -83,11 +99,12 @@ class HouseExport { def retMap = new ConcurrentHashMap() for(def i=0;i def data = [] data << m.city_code - data << m.houseId + //data << m.houseId + data << new ObjectId() data << "" data << m.city data << m.region_code @@ -248,18 +278,30 @@ class HouseExport { data << m.m_date==null?"":m.m_date data << m.source - datas << data + if(datas1.size()>2000000){ + datas2 << data + }else{ + datas1 << data + } + } + if(datas1.size()>0){ + datas << datas1 + } + + if(datas2.size()>0){ + datas << datas2 } return datas ; } - def createRentDatas(def list){ + def createRentDatas(def map){ def datas = [] - for(def m : list){ + map.each{k,m-> def data = [] data << m.city_code - data << m.houseId +// data << m.houseId + data << new ObjectId() data << "" data << m.city data << m.region_code @@ -278,6 +320,8 @@ class HouseExport { data << m.proptype==null?"":m.proptype data << m.propyears==null?"":m.propyears data << m.area==null?"":m.area + + data << m.avg_price==null?"":m.avg_price data << m.total_price==null?"":m.total_price data << m.rent_type_code==null?"":m.rent_type_code @@ -287,7 +331,12 @@ class HouseExport { data << m.estate_type==null?"":m.estate_type data << m.house_structure_code==null?"":m.house_structure_code data << m.build_structure_code==null?"":m.build_structure_code - data << m.area==null?"":m.area + if(m.rent_type_code && m.rent_type_code==1){ + data << "" + }else{ + data << m.area==null?"":m.area + } + data << "" data << m.house_type==null?"":m.house_type data << m.room==null?"":m.room @@ -318,43 +367,7 @@ class HouseExport { } - def batchCreateFile(ret, fileName, headers,charset){ - File file = new File(fileName); - if (file.exists()) { - file.delete(); - } - OutputStream out = new FileOutputStream(fileName); - def sb = new StringBuffer(); - - sb.append(headers.join("@|@")) - sb.append(System.getProperty("line.separator")); - - out.write(sb.toString().getBytes(charset)); - sb.setLength(0) - - def index = 0 - for(def row : ret){ - index++ - sb.append(StringUtils.join(row.toArray(), "|@|")) - sb.append(System.getProperty("line.separator")); - println index - - if(index%10000==0){ - println index+"开始写入CSV "+new Date().toLocaleString() - out.write(sb.toString().getBytes(charset)); - sb.setLength(0) - println index+"结束写入CSV "+new Date().toLocaleString() - } - - } - - if(sb.length()>0){ - out.write(sb.toString().getBytes(charset)); - sb.setLength(0) - } - out.close(); - } @@ -490,12 +503,12 @@ class HouseExport { } - def findHouses(def city, def cat, def startDate, def endDate, def name, def alias){ + def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){ QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false); QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); - QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", "sell")) + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) .must(date_range).must(area_range) if(!alias){ @@ -521,6 +534,13 @@ class HouseExport { def map = houses[k]; DBObject seawater = new BasicDBObject(map) + if(!seawater.total_price){ + continue; + } + if(!seawater.avg_price){ + continue; + } + if(!seawater.height){ seawater.height = null } @@ -534,8 +554,14 @@ class HouseExport { // if(!seawater.room){ // continue; // } - if(seawater.room && (seawater.room>=5 || seawater.room<1)){ - continue; + if(!seawater.room){ + seawater.room = null + } + + if(seawater.room){ + if(seawater.room>=5 || seawater.room<1){ + continue; + } } if(!seawater.hall){ @@ -551,28 +577,48 @@ class HouseExport { continue; } + def floor = parseFloor(seawater.floor_info) if(seawater.height && floor && floor.isNumber()){ if(floor.toInteger()>seawater.height){ floor = null seawater.height = null } + + } + + if(!floor){ + seawater.floor = null + } + if(floor && floor.toInteger()<1){ + floor = null } def rowMap = [:]; if(cat == "rent"){ rowMap.rent_type = seawater.rent_type + if(seawater.avg_price && seawater.avg_price>1000){ + continue + } + + if(seawater.avg_price && seawater.avg_price<10){ + continue + } + } rowMap.houseId = seawater.id rowMap.city = city - rowMap.region =seawater.region - rowMap.address = seawater.address + //rowMap.region =seawater.region + //rowMap.address = seawater.address rowMap.area = seawater.area rowMap.total_price = seawater.total_price rowMap.avg_price = seawater.avg_price rowMap.height = seawater.height=="null"?null:seawater.height + if(rowMap.height && rowMap.height<1){ + rowMap.height = null + } rowMap.floor = floor=="null"?null:floor rowMap.room = seawater.room=="null"?null:seawater.room rowMap.hall = seawater.hall=="null"?null:seawater.hall @@ -584,15 +630,36 @@ class HouseExport { def fitment = parseFitment(seawater.fitment) rowMap.fitment = fitment=="null"?null:fitment - rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") - rowMap.c_date = DateUtils.getConverDate(seawater.m_date) - rowMap.m_date = DateUtils.getConverDate(seawater.m_date) + + + if(seawater.p_date instanceof String){ + seawater.p_date = seawater.p_date.toInteger() + } + if(city=="哈尔滨" && seawater.p_date>=20181001 && seawater.p_date<20200201){ + rowMap.p_date = DateUtils.getRandomDate("2018-01-01", "2018-10-01") + rowMap.c_date = rowMap.p_date + rowMap.m_date = rowMap.p_date + }else{ + rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") + rowMap.c_date = DateUtils.getConverDate(seawater.m_date) + rowMap.m_date = DateUtils.getConverDate(seawater.m_date) + } + rets << rowMap } } return rets; } + def getPublicDate(def pDate){ + + if(pDate>=20181001 && pDate<20200201){ + return DateUtils.getRandomDate("2018-10-01", "2020-02-01") + } + def pdate = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(pDate), "yyyy-MM-dd HH:mm:ss") + return pdate; + } + static def parseFloor(def floorDesc){ if(floorDesc){ floorDesc = floorDesc.replaceAll("[\\s ]+", "") @@ -607,18 +674,18 @@ class HouseExport { if (matcher1.find()) { return matcher1.group(1) } - if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){ - return "中层" - } - - if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){ - return "高层" - } - - - if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){ - return "低层" - } +// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){ +// return "中层" +// } +// +// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){ +// return "高层" +// } +// +// +// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){ +// return "低层" +// } if(floorDesc.isNumber()){ return floorDesc; } @@ -700,6 +767,14 @@ class HouseExport { } + /* + * 生成一个用不重复的ID + */ +def GenNonDuplicateID(randomLength){ + return Number(Math.random().toString().substr(3,randomLength) + Date.now()).toString(36) +} +//GenNonDuplicateID()将生成 rfmipbs8ag0kgkcogc 类似的ID + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/AnjukeCrawlToDB.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/AnjukeCrawlToDB.groovy new file mode 100644 index 000000000..8c5009841 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/AnjukeCrawlToDB.groovy @@ -0,0 +1,199 @@ +package com.fangjia.script.gj.test + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import org.bson.types.ObjectId + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class AnjukeCrawlToDB { + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + + def totalCount = new AtomicLong() + + static main(args) { + def task = new AnjukeCrawlToDB(); + +// task.run() + + task.stat() + } + + def stat(){ + def districts = CommonDatautils.findAllDistricts("北京", "district", ["estate_charge","finish_building_date","build_area","household_count","car_site_count","cubage_percent","virescence_percent","estate_company","developer"]); + + def total = districts.size() + def estate_chargeCount = 0 + def build_areaCount = 0 + def household_countCount = 0 + def finish_building_dateCount = 0 + def car_site_countCount = 0 + def cubage_percentCount = 0 + def virescence_percentCount = 0 + def developerCount = 0 + def estate_companyCount = 0 + for(def dists : districts){ + if(dists.estate_charge){ + estate_chargeCount++ + } + // 总建面积 + if(dists.build_area){ + build_areaCount++ + } + // 总数 + if(dists.household_count){ + household_countCount++ + } + // 建造年代 + if(dists.finish_building_date){ + finish_building_dateCount++ + } + // 停车位 + if(dists.car_site_count){ + car_site_countCount++ + } + // 容积率 + if(dists.cubage_percent){ + cubage_percentCount++ + } + // 绿化率 + if(dists.virescence_percent){ + virescence_percentCount++ + } + // 开发商 + if(dists.developer){ + developerCount++ + } + // 物业公司 + if(dists.estate_company){ + estate_companyCount++ + } + } + + + def estate_charge_pre = DataHandleUtils.getDoubleRound(2, estate_chargeCount*100/total) + def build_area_pre = DataHandleUtils.getDoubleRound(2, build_areaCount*100/total) + def household_count_pre = DataHandleUtils.getDoubleRound(2, household_countCount*100/total) + def finish_building_date_pre = DataHandleUtils.getDoubleRound(2, finish_building_dateCount*100/total) + def car_site_pre = DataHandleUtils.getDoubleRound(2, car_site_countCount*100/total) + def cubage_percent_pre = DataHandleUtils.getDoubleRound(2, cubage_percentCount*100/total) + def virescence_percent_pre = DataHandleUtils.getDoubleRound(2, virescence_percentCount*100/total) + def developer_pre = DataHandleUtils.getDoubleRound(2, developerCount*100/total) + def estate_company_pre = DataHandleUtils.getDoubleRound(2, estate_companyCount*100/total) + + println finish_building_date_pre + println virescence_percent_pre + println cubage_percent_pre + + println build_area_pre + println estate_charge_pre + println developer_pre + println estate_company_pre +// println household_count_pre +// println car_site_pre + + + + + } + + + + + def run(){ + def datas = ExcelUtils.read("D:\\数据\\待抓取小区_安居客_detail.xlsx",0,1) + + + final CountDownLatch latch = new CountDownLatch(datas.size()) + for(def i=0;i0){ + udata.put("estate_charge",estateCharge) + } + } + // 总建面积 + if(!dists.build_area && data['L']){ + def buildArea = data['L'].toString().replace("m?","") + if(buildArea){ + udata.put("build_area",buildArea.toDouble().toInteger().toString()) + } + + } + // 总数 + if(!dists.household_count && data['M']){ + def householdCount= data['M'] + if(householdCount && householdCount.toString().isNumber()){ + udata.put("household_count",householdCount.toInteger()) + } + } + // 建造年代 + if(!dists.finish_building_date && data['N']){ + def year = data['N'].replace("年","") + if(year && year.isNumber()){ + udata.put("finish_building_date",year.toString()) + } + } + // 停车位 + if(!dists.car_site_count && data['O']){ + if(data['O'].isNumber()){ + udata.put("car_site_count", data['O'].toInteger()) + } + } + // 容积率 + if(!dists.cubage_percent && data['R']){ + def cubage_percent = data['R'] + if(cubage_percent && cubage_percent.isNumber() && cubage_percent.toDouble()<10){ + udata.put("cubage_percent", cubage_percent.toString()) + } + } + // 绿化率 + if(!dists.virescence_percent && data['Q']){ + def virescence_percent = data['Q'].toString().replace("%","") + if(virescence_percent && virescence_percent.isNumber()){ + udata.put("virescence_percent", virescence_percent.toString()) + } + } + // 开发商 + if(!dists.developer && data['R'] && !data['R'].contains("未知") && !data['R'].contains("无")){ + udata.put("developer", data['R']) + } + // 物业公司 + if(!dists.estate_company && data['S'] && !data['S'].contains("未知") && !data['S'].contains("无")){ + udata.put("estate_company",data['S']) + } + + if(udata){ + udata.put("m_date", new Date()) + println totalCount.get()+" "+data['A']+" "+udata + DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(data['A'])),new BasicDBObject('$set', udata)); + } + + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy index 8bc5d9198..a669411f2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy @@ -767,15 +767,19 @@ class DateUtils { } static String getConverDate(def mills){ - String s = null + Date date = null; if(mills instanceof String){ - s = mills.replace("T"," ").replace(".000Z","") + date = com.fangjia.core.util.DateUtil.parseISODate(mills) + Calendar cal = Calendar.getInstance(); + cal.setTime(date); + cal.add(Calendar.HOUR, 8);// 24小时制 + date = cal.getTime(); + //s = mills.replace("T"," ").replace(".000Z","") }else if(mills instanceof Long){ - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - Date date = new Date(mills); - s = dateFormat.format(date); + date = new Date(mills) } - + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + String s = dateFormat.format(date); return s } @@ -914,6 +918,13 @@ class DateUtils { SimpleDateFormat format = new SimpleDateFormat(pattern); return format.parse(s) } + + public static String getRandomDate(def startDate, def endDate){ + Date randomDate = randomDate(startDate, endDate); + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss"); + String result = format.format(randomDate); + return result; + } static main(args){ @@ -939,7 +950,48 @@ class DateUtils { //monthDiffCount() - println between(parseFromTightYYYYMMDD('2009-11-10'), parseFromTightYYYYMMDD('2008-12-12'), Calendar.MONTH) +// println between(parseFromTightYYYYMMDD('2009-11-10'), parseFromTightYYYYMMDD('2008-12-12'), Calendar.MONTH) + +// Date randomDate = randomDate("2010-09-20", "2011-09-22"); +// SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss"); +// String result = format.format(randomDate); + System.out.println(getRandomDate("2010-09-20", "2011-09-22")); + } + + + /** + * 生成随机时间 + * + * @param beginDate + * @param endDate + * @return + */ + public static Date randomDate(String beginDate, String endDate) { + try { + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd"); + Date start = format.parse(beginDate);// 构造开始日期 + Date end = format.parse(endDate);// 构造结束日期 + // getTime()表示返回自 1970 年 1 月 1 日 00:00:00 GMT 以来此 Date 对象表示的毫秒数。 + if (start.getTime() >= end.getTime()) { + return null; + } + long date = random(start.getTime(), end.getTime()); + return new Date(date); + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + public static long random(long begin, long end) { + long rtn = begin + (long) (Math.random() * (end - begin)); + // 如果返回的是开始时间和结束时间,则递归调用本函数查找随机值 + if (rtn == begin || rtn == end) { + return random(begin, end); + } + return rtn; + } + }