update
This commit is contained in:
+65
-18
@@ -65,15 +65,15 @@ class HouseExport {
|
||||
|
||||
def run(){
|
||||
|
||||
def citys = ["北京", "哈尔滨", "重庆"]
|
||||
def citys = [ "重庆", "北京"]
|
||||
for(def city : citys){
|
||||
getRegionCodeByCity(city)
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias","finish_building_date","house_property","property_life","build_type","estate_type2","build_structure"]);
|
||||
exportSellCSV(city, districts, 20170801, 20180901, 2017);
|
||||
exportRentCSV(city, districts, 20170801, 20180901, 2017);
|
||||
exportSellCSV(city, districts, 20170801, 20181001, 2017);
|
||||
exportRentCSV(city, districts, 20170801, 20181001, 2017);
|
||||
|
||||
exportSellCSV(city, districts, 20200201, 20210201, 2020);
|
||||
exportRentCSV(city, districts, 20200201, 20210201, 2020);
|
||||
exportSellCSV(city, districts, 20200201, 20210301, 2020);
|
||||
exportRentCSV(city, districts, 20200201, 20210301, 2020);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -87,11 +87,12 @@ class HouseExport {
|
||||
def retMap = new ConcurrentHashMap()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def rMap = districts[i];
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def ret = findHouses(city, "sell", startDate,endDate, rMap.name, rMap.alias)
|
||||
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias)
|
||||
if(ret){
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def regionCode = regionCodeMap.get(city +"-"+rMap.region)
|
||||
@@ -102,6 +103,7 @@ class HouseExport {
|
||||
def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure);
|
||||
for(def m : ret){
|
||||
m.city_code = cityCode
|
||||
m.region = rMap.region
|
||||
m.region_code = regionCode
|
||||
m.district_id = rMap.get("_id").toString()
|
||||
m.district_name = rMap.name
|
||||
@@ -112,6 +114,7 @@ class HouseExport {
|
||||
m.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall);
|
||||
m.direction_code = CcbCodeUtils.getDirectionCode(m.direction)
|
||||
m.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment);
|
||||
m.address = rMap.address
|
||||
m.build_structure_code = struCode
|
||||
m.house_structure_code = null//getHouseStructureCode();
|
||||
m.source = "fjw"
|
||||
@@ -154,7 +157,7 @@ class HouseExport {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def ret = findHouses(city, "rent", startDate,endDate, rMap.name, rMap.alias)
|
||||
def ret = findHouses(city, rMap.region,"rent", startDate,endDate, rMap.name, rMap.alias)
|
||||
if(ret){
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def regionCode = regionCodeMap.get(city +"-"+rMap.region)
|
||||
@@ -166,8 +169,10 @@ class HouseExport {
|
||||
for(def m : ret){
|
||||
m.city_code = cityCode
|
||||
m.region_code = regionCode
|
||||
m.region = rMap.region
|
||||
m.district_id = rMap.get("_id").toString()
|
||||
m.district_name = rMap.name
|
||||
m.address = rMap.address
|
||||
m.build_year = buildYear;
|
||||
m.proptype = housePro
|
||||
m.propyears = propertyYear;
|
||||
@@ -302,7 +307,12 @@ class HouseExport {
|
||||
data << m.build_year==null?"":m.build_year
|
||||
data << m.proptype==null?"":m.proptype
|
||||
data << m.propyears==null?"":m.propyears
|
||||
data << m.area==null?"":m.area
|
||||
if(m.rent_type_code && m.rent_type_code==1){
|
||||
data << m.area==null?"":m.area
|
||||
}else{
|
||||
data << ""
|
||||
}
|
||||
|
||||
data << m.avg_price==null?"":m.avg_price
|
||||
data << m.total_price==null?"":m.total_price
|
||||
data << m.rent_type_code==null?"":m.rent_type_code
|
||||
@@ -312,7 +322,12 @@ class HouseExport {
|
||||
data << m.estate_type==null?"":m.estate_type
|
||||
data << m.house_structure_code==null?"":m.house_structure_code
|
||||
data << m.build_structure_code==null?"":m.build_structure_code
|
||||
data << m.area==null?"":m.area
|
||||
if(m.rent_type_code && m.rent_type_code==1){
|
||||
data << ""
|
||||
}else{
|
||||
data << m.area==null?"":m.area
|
||||
}
|
||||
|
||||
data << ""
|
||||
data << m.house_type==null?"":m.house_type
|
||||
data << m.room==null?"":m.room
|
||||
@@ -479,12 +494,12 @@ class HouseExport {
|
||||
}
|
||||
|
||||
|
||||
def findHouses(def city, def cat, def startDate, def endDate, def name, def alias){
|
||||
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", cat))
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(date_range).must(area_range)
|
||||
|
||||
if(!alias){
|
||||
@@ -510,6 +525,13 @@ class HouseExport {
|
||||
def map = houses[k];
|
||||
DBObject seawater = new BasicDBObject(map)
|
||||
|
||||
if(!seawater.total_price){
|
||||
continue;
|
||||
}
|
||||
if(!seawater.avg_price){
|
||||
continue;
|
||||
}
|
||||
|
||||
if(!seawater.height){
|
||||
seawater.height = null
|
||||
}
|
||||
@@ -523,8 +545,14 @@ class HouseExport {
|
||||
// if(!seawater.room){
|
||||
// continue;
|
||||
// }
|
||||
if(seawater.room && (seawater.room>=5 || seawater.room<1)){
|
||||
continue;
|
||||
if(!seawater.room){
|
||||
seawater.room = null
|
||||
}
|
||||
|
||||
if(seawater.room){
|
||||
if(seawater.room>=5 || seawater.room<1){
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if(!seawater.hall){
|
||||
@@ -540,15 +568,21 @@ class HouseExport {
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
def floor = parseFloor(seawater.floor_info)
|
||||
if(seawater.height && floor && floor.isNumber()){
|
||||
if(floor.toInteger()>seawater.height){
|
||||
floor = null
|
||||
seawater.height = null
|
||||
}
|
||||
if(floor && floor.toInteger()<1){
|
||||
floor = null
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(!floor){
|
||||
seawater.floor = null
|
||||
}
|
||||
if(floor && floor.toInteger()<1){
|
||||
floor = null
|
||||
}
|
||||
|
||||
def rowMap = [:];
|
||||
@@ -557,11 +591,16 @@ class HouseExport {
|
||||
if(seawater.avg_price && seawater.avg_price>1000){
|
||||
continue
|
||||
}
|
||||
|
||||
if(seawater.avg_price && seawater.avg_price<10){
|
||||
continue
|
||||
}
|
||||
|
||||
}
|
||||
rowMap.houseId = seawater.id
|
||||
rowMap.city = city
|
||||
rowMap.region =seawater.region
|
||||
rowMap.address = seawater.address
|
||||
//rowMap.region =seawater.region
|
||||
//rowMap.address = seawater.address
|
||||
|
||||
|
||||
rowMap.area = seawater.area
|
||||
@@ -698,6 +737,14 @@ class HouseExport {
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
* 生成一个用不重复的ID
|
||||
*/
|
||||
def GenNonDuplicateID(randomLength){
|
||||
return Number(Math.random().toString().substr(3,randomLength) + Date.now()).toString(36)
|
||||
}
|
||||
//GenNonDuplicateID()将生成 rfmipbs8ag0kgkcogc 类似的ID
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
+199
@@ -0,0 +1,199 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class AnjukeCrawlToDB {
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
|
||||
static main(args) {
|
||||
def task = new AnjukeCrawlToDB();
|
||||
|
||||
// task.run()
|
||||
|
||||
task.stat()
|
||||
}
|
||||
|
||||
def stat(){
|
||||
def districts = CommonDatautils.findAllDistricts("北京", "district", ["estate_charge","finish_building_date","build_area","household_count","car_site_count","cubage_percent","virescence_percent","estate_company","developer"]);
|
||||
|
||||
def total = districts.size()
|
||||
def estate_chargeCount = 0
|
||||
def build_areaCount = 0
|
||||
def household_countCount = 0
|
||||
def finish_building_dateCount = 0
|
||||
def car_site_countCount = 0
|
||||
def cubage_percentCount = 0
|
||||
def virescence_percentCount = 0
|
||||
def developerCount = 0
|
||||
def estate_companyCount = 0
|
||||
for(def dists : districts){
|
||||
if(dists.estate_charge){
|
||||
estate_chargeCount++
|
||||
}
|
||||
// 总建面积
|
||||
if(dists.build_area){
|
||||
build_areaCount++
|
||||
}
|
||||
// 总数
|
||||
if(dists.household_count){
|
||||
household_countCount++
|
||||
}
|
||||
// 建造年代
|
||||
if(dists.finish_building_date){
|
||||
finish_building_dateCount++
|
||||
}
|
||||
// 停车位
|
||||
if(dists.car_site_count){
|
||||
car_site_countCount++
|
||||
}
|
||||
// 容积率
|
||||
if(dists.cubage_percent){
|
||||
cubage_percentCount++
|
||||
}
|
||||
// 绿化率
|
||||
if(dists.virescence_percent){
|
||||
virescence_percentCount++
|
||||
}
|
||||
// 开发商
|
||||
if(dists.developer){
|
||||
developerCount++
|
||||
}
|
||||
// 物业公司
|
||||
if(dists.estate_company){
|
||||
estate_companyCount++
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def estate_charge_pre = DataHandleUtils.getDoubleRound(2, estate_chargeCount*100/total)
|
||||
def build_area_pre = DataHandleUtils.getDoubleRound(2, build_areaCount*100/total)
|
||||
def household_count_pre = DataHandleUtils.getDoubleRound(2, household_countCount*100/total)
|
||||
def finish_building_date_pre = DataHandleUtils.getDoubleRound(2, finish_building_dateCount*100/total)
|
||||
def car_site_pre = DataHandleUtils.getDoubleRound(2, car_site_countCount*100/total)
|
||||
def cubage_percent_pre = DataHandleUtils.getDoubleRound(2, cubage_percentCount*100/total)
|
||||
def virescence_percent_pre = DataHandleUtils.getDoubleRound(2, virescence_percentCount*100/total)
|
||||
def developer_pre = DataHandleUtils.getDoubleRound(2, developerCount*100/total)
|
||||
def estate_company_pre = DataHandleUtils.getDoubleRound(2, estate_companyCount*100/total)
|
||||
|
||||
println finish_building_date_pre
|
||||
println virescence_percent_pre
|
||||
println cubage_percent_pre
|
||||
|
||||
println build_area_pre
|
||||
println estate_charge_pre
|
||||
println developer_pre
|
||||
println estate_company_pre
|
||||
// println household_count_pre
|
||||
// println car_site_pre
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def run(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\待抓取小区_安居客_detail.xlsx",0,1)
|
||||
|
||||
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
for(def i=0;i<datas.size();i++){
|
||||
def data = datas[i]
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
totalCount.addAndGet(1)
|
||||
findData(data);
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def findData(data){
|
||||
def dists = CommonDatautils.findDistrictById(data['A'])
|
||||
if(dists){
|
||||
def udata = new BasicDBObject()
|
||||
//物业费
|
||||
if(!dists.estate_charge && data['K']){
|
||||
def estateCharge = data['K'].replace("元/平米/月","")
|
||||
if(estateCharge && estateCharge.toDouble()>0){
|
||||
udata.put("estate_charge",estateCharge)
|
||||
}
|
||||
}
|
||||
// 总建面积
|
||||
if(!dists.build_area && data['L']){
|
||||
def buildArea = data['L'].toString().replace("m?","")
|
||||
if(buildArea){
|
||||
udata.put("build_area",buildArea.toDouble().toInteger().toString())
|
||||
}
|
||||
|
||||
}
|
||||
// 总数
|
||||
if(!dists.household_count && data['M']){
|
||||
def householdCount= data['M']
|
||||
if(householdCount && householdCount.toString().isNumber()){
|
||||
udata.put("household_count",householdCount.toInteger())
|
||||
}
|
||||
}
|
||||
// 建造年代
|
||||
if(!dists.finish_building_date && data['N']){
|
||||
def year = data['N'].replace("年","")
|
||||
if(year && year.isNumber()){
|
||||
udata.put("finish_building_date",year.toString())
|
||||
}
|
||||
}
|
||||
// 停车位
|
||||
if(!dists.car_site_count && data['O']){
|
||||
if(data['O'].isNumber()){
|
||||
udata.put("car_site_count", data['O'].toInteger())
|
||||
}
|
||||
}
|
||||
// 容积率
|
||||
if(!dists.cubage_percent && data['R']){
|
||||
def cubage_percent = data['R']
|
||||
if(cubage_percent && cubage_percent.isNumber() && cubage_percent.toDouble()<10){
|
||||
udata.put("cubage_percent", cubage_percent.toString())
|
||||
}
|
||||
}
|
||||
// 绿化率
|
||||
if(!dists.virescence_percent && data['Q']){
|
||||
def virescence_percent = data['Q'].toString().replace("%","")
|
||||
if(virescence_percent && virescence_percent.isNumber()){
|
||||
udata.put("virescence_percent", virescence_percent.toString())
|
||||
}
|
||||
}
|
||||
// 开发商
|
||||
if(!dists.developer && data['R'] && !data['R'].contains("未知") && !data['R'].contains("无")){
|
||||
udata.put("developer", data['R'])
|
||||
}
|
||||
// 物业公司
|
||||
if(!dists.estate_company && data['S'] && !data['S'].contains("未知") && !data['S'].contains("无")){
|
||||
udata.put("estate_company",data['S'])
|
||||
}
|
||||
|
||||
if(udata){
|
||||
udata.put("m_date", new Date())
|
||||
println totalCount.get()+" "+data['A']+" "+udata
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(data['A'])),new BasicDBObject('$set', udata));
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user