This commit is contained in:
2021-03-15 22:20:41 +08:00
parent 7be22202e3
commit 592c864233
2 changed files with 264 additions and 18 deletions
@@ -65,15 +65,15 @@ class HouseExport {
def run(){
def citys = ["北京", "哈尔滨", "重庆"]
def citys = [ "重庆", "北京"]
for(def city : citys){
getRegionCodeByCity(city)
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias","finish_building_date","house_property","property_life","build_type","estate_type2","build_structure"]);
exportSellCSV(city, districts, 20170801, 20180901, 2017);
exportRentCSV(city, districts, 20170801, 20180901, 2017);
exportSellCSV(city, districts, 20170801, 20181001, 2017);
exportRentCSV(city, districts, 20170801, 20181001, 2017);
exportSellCSV(city, districts, 20200201, 20210201, 2020);
exportRentCSV(city, districts, 20200201, 20210201, 2020);
exportSellCSV(city, districts, 20200201, 20210301, 2020);
exportRentCSV(city, districts, 20200201, 20210301, 2020);
}
}
@@ -87,11 +87,12 @@ class HouseExport {
def retMap = new ConcurrentHashMap()
for(def i=0;i<districts.size();i++){
final def rMap = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def ret = findHouses(city, "sell", startDate,endDate, rMap.name, rMap.alias)
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias)
if(ret){
def cityCode = cityCodeMap.get(city)
def regionCode = regionCodeMap.get(city +"-"+rMap.region)
@@ -102,6 +103,7 @@ class HouseExport {
def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure);
for(def m : ret){
m.city_code = cityCode
m.region = rMap.region
m.region_code = regionCode
m.district_id = rMap.get("_id").toString()
m.district_name = rMap.name
@@ -112,6 +114,7 @@ class HouseExport {
m.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall);
m.direction_code = CcbCodeUtils.getDirectionCode(m.direction)
m.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment);
m.address = rMap.address
m.build_structure_code = struCode
m.house_structure_code = null//getHouseStructureCode();
m.source = "fjw"
@@ -154,7 +157,7 @@ class HouseExport {
void run() {
try {
totalCount.addAndGet(1)
def ret = findHouses(city, "rent", startDate,endDate, rMap.name, rMap.alias)
def ret = findHouses(city, rMap.region,"rent", startDate,endDate, rMap.name, rMap.alias)
if(ret){
def cityCode = cityCodeMap.get(city)
def regionCode = regionCodeMap.get(city +"-"+rMap.region)
@@ -166,8 +169,10 @@ class HouseExport {
for(def m : ret){
m.city_code = cityCode
m.region_code = regionCode
m.region = rMap.region
m.district_id = rMap.get("_id").toString()
m.district_name = rMap.name
m.address = rMap.address
m.build_year = buildYear;
m.proptype = housePro
m.propyears = propertyYear;
@@ -302,7 +307,12 @@ class HouseExport {
data << m.build_year==null?"":m.build_year
data << m.proptype==null?"":m.proptype
data << m.propyears==null?"":m.propyears
data << m.area==null?"":m.area
if(m.rent_type_code && m.rent_type_code==1){
data << m.area==null?"":m.area
}else{
data << ""
}
data << m.avg_price==null?"":m.avg_price
data << m.total_price==null?"":m.total_price
data << m.rent_type_code==null?"":m.rent_type_code
@@ -312,7 +322,12 @@ class HouseExport {
data << m.estate_type==null?"":m.estate_type
data << m.house_structure_code==null?"":m.house_structure_code
data << m.build_structure_code==null?"":m.build_structure_code
data << m.area==null?"":m.area
if(m.rent_type_code && m.rent_type_code==1){
data << ""
}else{
data << m.area==null?"":m.area
}
data << ""
data << m.house_type==null?"":m.house_type
data << m.room==null?"":m.room
@@ -479,12 +494,12 @@ class HouseExport {
}
def findHouses(def city, def cat, def startDate, def endDate, def name, def alias){
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", cat))
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range)
if(!alias){
@@ -510,6 +525,13 @@ class HouseExport {
def map = houses[k];
DBObject seawater = new BasicDBObject(map)
if(!seawater.total_price){
continue;
}
if(!seawater.avg_price){
continue;
}
if(!seawater.height){
seawater.height = null
}
@@ -523,8 +545,14 @@ class HouseExport {
// if(!seawater.room){
// continue;
// }
if(seawater.room && (seawater.room>=5 || seawater.room<1)){
continue;
if(!seawater.room){
seawater.room = null
}
if(seawater.room){
if(seawater.room>=5 || seawater.room<1){
continue;
}
}
if(!seawater.hall){
@@ -540,15 +568,21 @@ class HouseExport {
continue;
}
def floor = parseFloor(seawater.floor_info)
if(seawater.height && floor && floor.isNumber()){
if(floor.toInteger()>seawater.height){
floor = null
seawater.height = null
}
if(floor && floor.toInteger()<1){
floor = null
}
}
if(!floor){
seawater.floor = null
}
if(floor && floor.toInteger()<1){
floor = null
}
def rowMap = [:];
@@ -557,11 +591,16 @@ class HouseExport {
if(seawater.avg_price && seawater.avg_price>1000){
continue
}
if(seawater.avg_price && seawater.avg_price<10){
continue
}
}
rowMap.houseId = seawater.id
rowMap.city = city
rowMap.region =seawater.region
rowMap.address = seawater.address
//rowMap.region =seawater.region
//rowMap.address = seawater.address
rowMap.area = seawater.area
@@ -698,6 +737,14 @@ class HouseExport {
}
/*
* 生成一个用不重复的ID
*/
def GenNonDuplicateID(randomLength){
return Number(Math.random().toString().substr(3,randomLength) + Date.now()).toString(36)
}
//GenNonDuplicateID()将生成 rfmipbs8ag0kgkcogc 类似的ID
}
@@ -0,0 +1,199 @@
package com.fangjia.script.gj.test
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import org.bson.types.ObjectId
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class AnjukeCrawlToDB {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
def totalCount = new AtomicLong()
static main(args) {
def task = new AnjukeCrawlToDB();
// task.run()
task.stat()
}
def stat(){
def districts = CommonDatautils.findAllDistricts("北京", "district", ["estate_charge","finish_building_date","build_area","household_count","car_site_count","cubage_percent","virescence_percent","estate_company","developer"]);
def total = districts.size()
def estate_chargeCount = 0
def build_areaCount = 0
def household_countCount = 0
def finish_building_dateCount = 0
def car_site_countCount = 0
def cubage_percentCount = 0
def virescence_percentCount = 0
def developerCount = 0
def estate_companyCount = 0
for(def dists : districts){
if(dists.estate_charge){
estate_chargeCount++
}
// 总建面积
if(dists.build_area){
build_areaCount++
}
// 总数
if(dists.household_count){
household_countCount++
}
// 建造年代
if(dists.finish_building_date){
finish_building_dateCount++
}
// 停车位
if(dists.car_site_count){
car_site_countCount++
}
// 容积率
if(dists.cubage_percent){
cubage_percentCount++
}
// 绿化率
if(dists.virescence_percent){
virescence_percentCount++
}
// 开发商
if(dists.developer){
developerCount++
}
// 物业公司
if(dists.estate_company){
estate_companyCount++
}
}
def estate_charge_pre = DataHandleUtils.getDoubleRound(2, estate_chargeCount*100/total)
def build_area_pre = DataHandleUtils.getDoubleRound(2, build_areaCount*100/total)
def household_count_pre = DataHandleUtils.getDoubleRound(2, household_countCount*100/total)
def finish_building_date_pre = DataHandleUtils.getDoubleRound(2, finish_building_dateCount*100/total)
def car_site_pre = DataHandleUtils.getDoubleRound(2, car_site_countCount*100/total)
def cubage_percent_pre = DataHandleUtils.getDoubleRound(2, cubage_percentCount*100/total)
def virescence_percent_pre = DataHandleUtils.getDoubleRound(2, virescence_percentCount*100/total)
def developer_pre = DataHandleUtils.getDoubleRound(2, developerCount*100/total)
def estate_company_pre = DataHandleUtils.getDoubleRound(2, estate_companyCount*100/total)
println finish_building_date_pre
println virescence_percent_pre
println cubage_percent_pre
println build_area_pre
println estate_charge_pre
println developer_pre
println estate_company_pre
// println household_count_pre
// println car_site_pre
}
def run(){
def datas = ExcelUtils.read("D:\\数据\\待抓取小区_安居客_detail.xlsx",0,1)
final CountDownLatch latch = new CountDownLatch(datas.size())
for(def i=0;i<datas.size();i++){
def data = datas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
totalCount.addAndGet(1)
findData(data);
}
});
}
}
def findData(data){
def dists = CommonDatautils.findDistrictById(data['A'])
if(dists){
def udata = new BasicDBObject()
//物业费
if(!dists.estate_charge && data['K']){
def estateCharge = data['K'].replace("元/平米/月","")
if(estateCharge && estateCharge.toDouble()>0){
udata.put("estate_charge",estateCharge)
}
}
// 总建面积
if(!dists.build_area && data['L']){
def buildArea = data['L'].toString().replace("m?","")
if(buildArea){
udata.put("build_area",buildArea.toDouble().toInteger().toString())
}
}
// 总数
if(!dists.household_count && data['M']){
def householdCount= data['M']
if(householdCount && householdCount.toString().isNumber()){
udata.put("household_count",householdCount.toInteger())
}
}
// 建造年代
if(!dists.finish_building_date && data['N']){
def year = data['N'].replace("年","")
if(year && year.isNumber()){
udata.put("finish_building_date",year.toString())
}
}
// 停车位
if(!dists.car_site_count && data['O']){
if(data['O'].isNumber()){
udata.put("car_site_count", data['O'].toInteger())
}
}
// 容积率
if(!dists.cubage_percent && data['R']){
def cubage_percent = data['R']
if(cubage_percent && cubage_percent.isNumber() && cubage_percent.toDouble()<10){
udata.put("cubage_percent", cubage_percent.toString())
}
}
// 绿化率
if(!dists.virescence_percent && data['Q']){
def virescence_percent = data['Q'].toString().replace("%","")
if(virescence_percent && virescence_percent.isNumber()){
udata.put("virescence_percent", virescence_percent.toString())
}
}
// 开发商
if(!dists.developer && data['R'] && !data['R'].contains("未知") && !data['R'].contains("无")){
udata.put("developer", data['R'])
}
// 物业公司
if(!dists.estate_company && data['S'] && !data['S'].contains("未知") && !data['S'].contains("无")){
udata.put("estate_company",data['S'])
}
if(udata){
udata.put("m_date", new Date())
println totalCount.get()+" "+data['A']+" "+udata
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(data['A'])),new BasicDBObject('$set', udata));
}
}
}
}