Merge branch 'script_gj' into 'master'

去别墅

See merge request !90
This commit is contained in:
2021-08-25 14:00:58 +08:00
4 changed files with 205 additions and 50 deletions
@@ -13,6 +13,7 @@ import com.fangjia.script.util.FangJiaAreaMapUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.sql.Sql
import org.apache.commons.lang.StringUtils
import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
@@ -53,6 +54,7 @@ class BigdataForsaleBatch extends BaseBatch{
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
}
public static final String[] TAGS = {"独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"};
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
@@ -346,6 +348,10 @@ class BigdataForsaleBatch extends BaseBatch{
if(floor && floor.toInteger()<1){
floor = null
}
def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
if(vallaFlag){
continue;
}
def rowMap = [:];
if(cat == "rent"){
@@ -410,6 +416,39 @@ class BigdataForsaleBatch extends BaseBatch{
return rets;
}
/**
* 是否是别墅
* @return
*/
public boolean isVilla(area, name, height, room) {
if (area == null) {
return false;
}
if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) {
return true;
}
boolean ret = false;
if (height != null && room != null) {
if (height <= 3) {
if (area > 200 && room >= 5) {
ret = true;
}
}
} else if (height == null && room != null) {
if (area > 360 && room >= 6) {
ret = true;
}
} else if (height == null && room == null) {
if (area > 500) {
ret = true;
}
}
return ret;
}
def getPublicDate(def pDate){
if(pDate>=20181001 && pDate<20200201){
@@ -1,5 +1,6 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
@@ -8,12 +9,13 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import org.bson.types.ObjectId
/**
* 建行数据poi同步
*/
@@ -31,31 +33,35 @@ class BigdataPoiBatch extends BaseBatch{
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def citys = CcbCommonUtils.citys;
// def path = "D:\\数据\\建行\\房产估值\\poi\\公交.xlsx"
def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1)
def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1)
def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\医院.xlsx", 0,1)
def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1)
def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1)
def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\学校.xlsx", 0,1)
// def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1)
// def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1)
// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\医院.xlsx", 0,1)
// def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1)
// def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1)
// def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\学校.xlsx", 0,1)
def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\中学.xlsx", 0,1)
def rets = []
def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁")
def rows1 = createDatas(citys, cityCodeDataMap, hosptialDatas, "学校")
rets.addAll(rows1)
def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交")
rets.addAll(rows2)
// def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁")
// rets.addAll(rows1)
//
def rows3 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院")
rets.addAll(rows3)
// def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交")
// rets.addAll(rows2)
//
def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行")
rets.addAll(rows4)
def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务")
rets.addAll(rows5)
def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校")
rets.addAll(rows6)
// def rows3 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院")
// rets.addAll(rows3)
//
// def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行")
// rets.addAll(rows4)
//
// def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务")
// rets.addAll(rows5)
//
// def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校")
// rets.addAll(rows6)
def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","来源公司"]
@@ -113,7 +119,7 @@ class BigdataPoiBatch extends BaseBatch{
void run() {
try{
totalCount.addAndGet(1)
if(citys.contains(d['D'])){
if(citys.contains(d['A'])){
def ret = addValue(d, cityCodeDataMap, cityDistMap, type)
if(ret){
rets << ret
@@ -133,24 +139,49 @@ class BigdataPoiBatch extends BaseBatch{
return rets;
}
def addValue(d, cityCodeDataMap, cityDistMap, type){
def getRegionByLocation(lng, lat){
def jsonObject = GaoDeApiUtils.geocode(lng.toString()+","+lat.toString())
if(jsonObject && jsonObject.info=="OK"){
def compent = jsonObject.regeocode.addressComponent
if(compent){
def district = compent.district
return district;
}
}
return null
}
def addValue2(d, cityCodeDataMap, cityDistMap, type){
def ret = []
def citycode = cityCodeDataMap.get(d['D'])
def citycode = cityCodeDataMap.get(d['E'])
ret << citycode
def distCodeMap = cityDistMap.get(d['D'])
if(d['E']){
def distObj = distCodeMap.get(d['E'])
def region =getRegionByLocation(d['H'], d['I'])
def distCodeMap = cityDistMap.get(d['E'])
if(region){
region = RegionData.formatRegion(d['E'], region)
}
if(region){
def distObj = distCodeMap.get(region)
if(distObj){
ret << distObj.district_code
}else{
ret << ""
println d['D']+" " + d['E']+" 无对应的区域编码 "+d['F']
println d['E']+" " + region+" 无对应的区域编码 "+d['F']
}
}else{
ret << ""
}
ret << d['A']
ret << d['F']
if(d['A']){
ret << d['A']
}else{
ret << new ObjectId()
}
ret << d['G']
ret << type
if(type=="地铁"){
@@ -192,4 +223,67 @@ class BigdataPoiBatch extends BaseBatch{
return ret;
}
def addValue(d, cityCodeDataMap, cityDistMap, type){
def ret = []
def citycode = cityCodeDataMap.get(d['A'])
ret << citycode
def distCodeMap = cityDistMap.get(d['A'])
def region =RegionData.formatRegion(d['A'], d['B'])
if(region){
def distObj = distCodeMap.get(region)
if(distObj){
ret << distObj.district_code
}else{
ret << ""
println d['A']+" " + d['B']+" 无对应的区域编码 "+d['F']
}
}else{
ret << ""
}
ret << new ObjectId()
ret << d['C']
ret << type
if(type=="地铁"){
ret << "cz125"
}else if(type=="公交"){
ret << "cz123"
}else if(type=="医院"){
ret << "hp111"
}else if(type == "银行"){
ret << "se118"
}else if(type =="丧葬服务"){
ret << "ob101"
}else if(type =="学校"){
// if(d['F'].contains("中学")){
ret << "ed138"
// }else if(d['F'].contains("小学")){
// ret << "ed114"
// }
}
if(d['I']){
def lat = d['J'].toDouble()
def lng = d['I'].toDouble()
def loactionMap = GeoUtils.gcjToGps84(lat, lng)
if(loactionMap){
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
}else{
ret << ""
ret << ""
}
}else{
ret << ""
ret << ""
}
ret << ""
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
return ret;
}
}
@@ -23,8 +23,8 @@ class TradeStatLog {
static main(args) {
def task = new TradeStatLog()
//task.run()
task.thirdPartyStat()
task.run()
// task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
}
@@ -32,33 +32,40 @@ class TradeStatLog {
def run(){
def citys = CcbCommonUtils.citys;
def rets = []
rets << ["城市","5月总量","6月总量","贝壳","链家", "我爱我家","房天下","中原"]
rets << ["城市","5月总量","6月总量","7月总量","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
def startDate = "2021-05-01"
def date = "2021-06-01"
def endDate = "2021-07-01"
def newDate = "2021-08-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
def curMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
def nearMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(newDate,"yyyy-MM-dd"))
def ret = []
ret << city
if(city == "上海"){
def lastMonthYoudaCount = getYouDaData(DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
def curMonthYoudaCount = getYouDaData(DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
def nearMonthYoudaCount = getYouDaData(DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(newDate,"yyyy-MM-dd"))
def a = lastMonthCount.toInteger() + lastMonthYoudaCount.toInteger()
def b = curMonthCount.toInteger() + curMonthYoudaCount.toInteger()
def c = nearMonthCount.toInteger() + nearMonthYoudaCount.toInteger()
ret << a
ret << b
def c = (b-a)*100/a
def pre = DataHandleUtils.getDoubleRound(2, c)
ret << c
def d = (c-b)*100/a
def pre = DataHandleUtils.getDoubleRound(2, d)
ret << pre
}else{
ret << lastMonthCount
ret << curMonthCount
ret << nearMonthCount
if(lastMonthCount){
def a = (curMonthCount-lastMonthCount)*100/lastMonthCount
def a = (nearMonthCount-curMonthCount)*100/curMonthCount
def pre = DataHandleUtils.getDoubleRound(2, a)
ret << pre
}else{
@@ -70,8 +77,10 @@ class TradeStatLog {
for(def source : sources){
def sourceLastMonthCount = findTradesCount(city, source, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
def sourceCurMonthCount = findTradesCount(city, source, DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
def sourceNearMonthCount = findTradesCount(city, source, DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(newDate,"yyyy-MM-dd"))
ret << sourceLastMonthCount
ret << sourceCurMonthCount
ret << sourceNearMonthCount
}
rets << ret
println ret
@@ -4,6 +4,7 @@ import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.mongodb.BasicDBObject
import org.apache.solr.common.util.Hash
import org.bson.types.ObjectId
/**
* 福州农商行
@@ -16,7 +17,7 @@ class SynchDistrictPrice {
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
// "anjuke.com":"安居客","cityhouse.cn":"城市房产"]
def sourceMap = ["zhuge.com":"诸葛找房"]
def sourceMap = ["lianjia.com":"链家"]
static main(args) {
def task = new SynchDistrictPrice()
task.run()
@@ -27,11 +28,13 @@ class SynchDistrictPrice {
def citys = CommonUtils.findAllCity()
sourceMap.each { k, v ->
for(def city : citys){
def rets = findCrawlDistrictDetails(city, k, v)
println city +" "+rets.size()
synchData(city, rets, v)
for(def city : citys){
if(!["上海","北京","海口"].contains(city)){
def rets = findCrawlDistrictDetails(city, k, v)
println city +" "+rets.size()
synchData(city, rets, v)
}
}
}
@@ -77,6 +80,7 @@ class SynchDistrictPrice {
q.put("city", city)
def count = thirdPartyColl.count(q)
def addIndex = 0
def updateIndex = 0
if(count==0){
try{
thirdPartyColl.insert(rets)
@@ -86,13 +90,25 @@ class SynchDistrictPrice {
println "["+source +"-"+city + "] add:"+ rets.size()+" 原抓取:"+count
}else{
for(def r : rets){
def flag = findIsExists(r.city, r.region, r.name, SHORT_DATE, source)
if(!flag){
def obj= findIsExists(r.city, r.region, r.name, SHORT_DATE, source)
if(obj){
thirdPartyColl.remove(new BasicDBObject('_id',obj._id))
addIndex++
thirdPartyColl.save(r)
}
// if(!obj){
// addIndex++
// thirdPartyColl.save(r)
// }else{
// if(r.avg_price && !obj.avg_price){
// def udata = new BasicDBObject()
// udata.put("avg_price", r.avg_price)
// thirdPartyColl.update(new BasicDBObject('_id',obj._id),new BasicDBObject('$set',udata))
// updateIndex++
// }
// }
}
println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count
println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count +" 修改:"+updateIndex
}
@@ -105,10 +121,7 @@ class SynchDistrictPrice {
q.put("city", city)
q.put("region", region)
q.put("name", name)
def count = thirdPartyColl.count(q)
if(count){
return true
}
return false
def obj = thirdPartyColl.findOne(q)
return obj
}
}