去别墅

This commit is contained in:
2021-08-25 14:44:59 +08:00
parent 125b48b17f
commit df2227edcf
4 changed files with 205 additions and 50 deletions
@@ -13,6 +13,7 @@ import com.fangjia.script.util.FangJiaAreaMapUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBObject import com.mongodb.DBObject
import groovy.sql.Sql import groovy.sql.Sql
import org.apache.commons.lang.StringUtils
import org.bson.types.ObjectId import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilder
@@ -53,6 +54,7 @@ class BigdataForsaleBatch extends BaseBatch{
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"]) FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
} }
public static final String[] TAGS = {"独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"};
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
@@ -346,6 +348,10 @@ class BigdataForsaleBatch extends BaseBatch{
if(floor && floor.toInteger()<1){ if(floor && floor.toInteger()<1){
floor = null floor = null
} }
def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
if(vallaFlag){
continue;
}
def rowMap = [:]; def rowMap = [:];
if(cat == "rent"){ if(cat == "rent"){
@@ -410,6 +416,39 @@ class BigdataForsaleBatch extends BaseBatch{
return rets; return rets;
} }
/**
* 是否是别墅
* @return
*/
public boolean isVilla(area, name, height, room) {
if (area == null) {
return false;
}
if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) {
return true;
}
boolean ret = false;
if (height != null && room != null) {
if (height <= 3) {
if (area > 200 && room >= 5) {
ret = true;
}
}
} else if (height == null && room != null) {
if (area > 360 && room >= 6) {
ret = true;
}
} else if (height == null && room == null) {
if (area > 500) {
ret = true;
}
}
return ret;
}
def getPublicDate(def pDate){ def getPublicDate(def pDate){
if(pDate>=20181001 && pDate<20200201){ if(pDate>=20181001 && pDate<20200201){
@@ -1,5 +1,6 @@
package com.fangjia.script.ccbevaluation package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.core.util.GeoUtils import com.fangjia.core.util.GeoUtils
import com.fangjia.script.BaseBatch import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner import com.fangjia.script.BatchOwner
@@ -8,12 +9,13 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import java.util.concurrent.CountDownLatch import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong import java.util.concurrent.atomic.AtomicLong
import org.bson.types.ObjectId
/** /**
* 建行数据poi同步 * 建行数据poi同步
*/ */
@@ -31,31 +33,35 @@ class BigdataPoiBatch extends BaseBatch{
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def citys = CcbCommonUtils.citys; def citys = CcbCommonUtils.citys;
// def path = "D:\\数据\\建行\\房产估值\\poi\\公交.xlsx" // def path = "D:\\数据\\建行\\房产估值\\poi\\公交.xlsx"
def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1) // def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1)
def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1) // def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1)
def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\医院.xlsx", 0,1) // def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\医院.xlsx", 0,1)
def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1) // def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1)
def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1) // def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1)
def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\学校.xlsx", 0,1) // def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\学校.xlsx", 0,1)
def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\中学.xlsx", 0,1)
def rets = [] def rets = []
def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁")
def rows1 = createDatas(citys, cityCodeDataMap, hosptialDatas, "学校")
rets.addAll(rows1) rets.addAll(rows1)
def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交") // def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁")
rets.addAll(rows2) // rets.addAll(rows1)
// //
def rows3 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院") // def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交")
rets.addAll(rows3) // rets.addAll(rows2)
// //
def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行") // def rows3 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院")
rets.addAll(rows4) // rets.addAll(rows3)
//
def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务") // def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行")
rets.addAll(rows5) // rets.addAll(rows4)
//
def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校") // def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务")
rets.addAll(rows6) // rets.addAll(rows5)
//
// def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校")
// rets.addAll(rows6)
def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","来源公司"] def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","来源公司"]
@@ -113,7 +119,7 @@ class BigdataPoiBatch extends BaseBatch{
void run() { void run() {
try{ try{
totalCount.addAndGet(1) totalCount.addAndGet(1)
if(citys.contains(d['D'])){ if(citys.contains(d['A'])){
def ret = addValue(d, cityCodeDataMap, cityDistMap, type) def ret = addValue(d, cityCodeDataMap, cityDistMap, type)
if(ret){ if(ret){
rets << ret rets << ret
@@ -133,24 +139,49 @@ class BigdataPoiBatch extends BaseBatch{
return rets; return rets;
} }
def addValue(d, cityCodeDataMap, cityDistMap, type){ def getRegionByLocation(lng, lat){
def jsonObject = GaoDeApiUtils.geocode(lng.toString()+","+lat.toString())
if(jsonObject && jsonObject.info=="OK"){
def compent = jsonObject.regeocode.addressComponent
if(compent){
def district = compent.district
return district;
}
}
return null
}
def addValue2(d, cityCodeDataMap, cityDistMap, type){
def ret = [] def ret = []
def citycode = cityCodeDataMap.get(d['D']) def citycode = cityCodeDataMap.get(d['E'])
ret << citycode ret << citycode
def distCodeMap = cityDistMap.get(d['D'])
if(d['E']){ def region =getRegionByLocation(d['H'], d['I'])
def distObj = distCodeMap.get(d['E']) def distCodeMap = cityDistMap.get(d['E'])
if(region){
region = RegionData.formatRegion(d['E'], region)
}
if(region){
def distObj = distCodeMap.get(region)
if(distObj){ if(distObj){
ret << distObj.district_code ret << distObj.district_code
}else{ }else{
ret << "" ret << ""
println d['D']+" " + d['E']+" 无对应的区域编码 "+d['F'] println d['E']+" " + region+" 无对应的区域编码 "+d['F']
} }
}else{ }else{
ret << "" ret << ""
} }
ret << d['A']
ret << d['F'] if(d['A']){
ret << d['A']
}else{
ret << new ObjectId()
}
ret << d['G']
ret << type ret << type
if(type=="地铁"){ if(type=="地铁"){
@@ -192,4 +223,67 @@ class BigdataPoiBatch extends BaseBatch{
return ret; return ret;
} }
def addValue(d, cityCodeDataMap, cityDistMap, type){
def ret = []
def citycode = cityCodeDataMap.get(d['A'])
ret << citycode
def distCodeMap = cityDistMap.get(d['A'])
def region =RegionData.formatRegion(d['A'], d['B'])
if(region){
def distObj = distCodeMap.get(region)
if(distObj){
ret << distObj.district_code
}else{
ret << ""
println d['A']+" " + d['B']+" 无对应的区域编码 "+d['F']
}
}else{
ret << ""
}
ret << new ObjectId()
ret << d['C']
ret << type
if(type=="地铁"){
ret << "cz125"
}else if(type=="公交"){
ret << "cz123"
}else if(type=="医院"){
ret << "hp111"
}else if(type == "银行"){
ret << "se118"
}else if(type =="丧葬服务"){
ret << "ob101"
}else if(type =="学校"){
// if(d['F'].contains("中学")){
ret << "ed138"
// }else if(d['F'].contains("小学")){
// ret << "ed114"
// }
}
if(d['I']){
def lat = d['J'].toDouble()
def lng = d['I'].toDouble()
def loactionMap = GeoUtils.gcjToGps84(lat, lng)
if(loactionMap){
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
}else{
ret << ""
ret << ""
}
}else{
ret << ""
ret << ""
}
ret << ""
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
return ret;
}
} }
@@ -23,8 +23,8 @@ class TradeStatLog {
static main(args) { static main(args) {
def task = new TradeStatLog() def task = new TradeStatLog()
//task.run() task.run()
task.thirdPartyStat() // task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR) // println Calendar.getInstance().get(Calendar.YEAR)
} }
@@ -32,33 +32,40 @@ class TradeStatLog {
def run(){ def run(){
def citys = CcbCommonUtils.citys; def citys = CcbCommonUtils.citys;
def rets = [] def rets = []
rets << ["城市","5月总量","6月总量","贝壳","链家", "我爱我家","房天下","中原"] rets << ["城市","5月总量","6月总量","7月总量","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"] def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
def startDate = "2021-05-01" def startDate = "2021-05-01"
def date = "2021-06-01" def date = "2021-06-01"
def endDate = "2021-07-01" def endDate = "2021-07-01"
def newDate = "2021-08-01"
for(def city : citys){ for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
def curMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd")) def curMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
def nearMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(newDate,"yyyy-MM-dd"))
def ret = [] def ret = []
ret << city ret << city
if(city == "上海"){ if(city == "上海"){
def lastMonthYoudaCount = getYouDaData(DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) def lastMonthYoudaCount = getYouDaData(DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
def curMonthYoudaCount = getYouDaData(DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd")) def curMonthYoudaCount = getYouDaData(DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
def nearMonthYoudaCount = getYouDaData(DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(newDate,"yyyy-MM-dd"))
def a = lastMonthCount.toInteger() + lastMonthYoudaCount.toInteger() def a = lastMonthCount.toInteger() + lastMonthYoudaCount.toInteger()
def b = curMonthCount.toInteger() + curMonthYoudaCount.toInteger() def b = curMonthCount.toInteger() + curMonthYoudaCount.toInteger()
def c = nearMonthCount.toInteger() + nearMonthYoudaCount.toInteger()
ret << a ret << a
ret << b ret << b
def c = (b-a)*100/a ret << c
def pre = DataHandleUtils.getDoubleRound(2, c) def d = (c-b)*100/a
def pre = DataHandleUtils.getDoubleRound(2, d)
ret << pre ret << pre
}else{ }else{
ret << lastMonthCount ret << lastMonthCount
ret << curMonthCount ret << curMonthCount
ret << nearMonthCount
if(lastMonthCount){ if(lastMonthCount){
def a = (curMonthCount-lastMonthCount)*100/lastMonthCount def a = (nearMonthCount-curMonthCount)*100/curMonthCount
def pre = DataHandleUtils.getDoubleRound(2, a) def pre = DataHandleUtils.getDoubleRound(2, a)
ret << pre ret << pre
}else{ }else{
@@ -70,8 +77,10 @@ class TradeStatLog {
for(def source : sources){ for(def source : sources){
def sourceLastMonthCount = findTradesCount(city, source, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) def sourceLastMonthCount = findTradesCount(city, source, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
def sourceCurMonthCount = findTradesCount(city, source, DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd")) def sourceCurMonthCount = findTradesCount(city, source, DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
def sourceNearMonthCount = findTradesCount(city, source, DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(newDate,"yyyy-MM-dd"))
ret << sourceLastMonthCount ret << sourceLastMonthCount
ret << sourceCurMonthCount ret << sourceCurMonthCount
ret << sourceNearMonthCount
} }
rets << ret rets << ret
println ret println ret
@@ -4,6 +4,7 @@ import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DateUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import org.apache.solr.common.util.Hash import org.apache.solr.common.util.Hash
import org.bson.types.ObjectId
/** /**
* 福州农商行 * 福州农商行
@@ -16,7 +17,7 @@ class SynchDistrictPrice {
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl(); def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房", // def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
// "anjuke.com":"安居客","cityhouse.cn":"城市房产"] // "anjuke.com":"安居客","cityhouse.cn":"城市房产"]
def sourceMap = ["zhuge.com":"诸葛找房"] def sourceMap = ["lianjia.com":"链家"]
static main(args) { static main(args) {
def task = new SynchDistrictPrice() def task = new SynchDistrictPrice()
task.run() task.run()
@@ -27,11 +28,13 @@ class SynchDistrictPrice {
def citys = CommonUtils.findAllCity() def citys = CommonUtils.findAllCity()
sourceMap.each { k, v -> sourceMap.each { k, v ->
for(def city : citys){
def rets = findCrawlDistrictDetails(city, k, v)
println city +" "+rets.size()
synchData(city, rets, v)
for(def city : citys){
if(!["上海","北京","海口"].contains(city)){
def rets = findCrawlDistrictDetails(city, k, v)
println city +" "+rets.size()
synchData(city, rets, v)
}
} }
} }
@@ -77,6 +80,7 @@ class SynchDistrictPrice {
q.put("city", city) q.put("city", city)
def count = thirdPartyColl.count(q) def count = thirdPartyColl.count(q)
def addIndex = 0 def addIndex = 0
def updateIndex = 0
if(count==0){ if(count==0){
try{ try{
thirdPartyColl.insert(rets) thirdPartyColl.insert(rets)
@@ -86,13 +90,25 @@ class SynchDistrictPrice {
println "["+source +"-"+city + "] add:"+ rets.size()+" 原抓取:"+count println "["+source +"-"+city + "] add:"+ rets.size()+" 原抓取:"+count
}else{ }else{
for(def r : rets){ for(def r : rets){
def flag = findIsExists(r.city, r.region, r.name, SHORT_DATE, source) def obj= findIsExists(r.city, r.region, r.name, SHORT_DATE, source)
if(!flag){ if(obj){
thirdPartyColl.remove(new BasicDBObject('_id',obj._id))
addIndex++ addIndex++
thirdPartyColl.save(r) thirdPartyColl.save(r)
} }
// if(!obj){
// addIndex++
// thirdPartyColl.save(r)
// }else{
// if(r.avg_price && !obj.avg_price){
// def udata = new BasicDBObject()
// udata.put("avg_price", r.avg_price)
// thirdPartyColl.update(new BasicDBObject('_id',obj._id),new BasicDBObject('$set',udata))
// updateIndex++
// }
// }
} }
println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count +" 修改:"+updateIndex
} }
@@ -105,10 +121,7 @@ class SynchDistrictPrice {
q.put("city", city) q.put("city", city)
q.put("region", region) q.put("region", region)
q.put("name", name) q.put("name", name)
def count = thirdPartyColl.count(q) def obj = thirdPartyColl.findOne(q)
if(count){ return obj
return true
}
return false
} }
} }