From df2227edcffdbaf91f2da9e2d0d1d2040b0b9f2b Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 25 Aug 2021 14:44:59 +0800 Subject: [PATCH] =?UTF-8?q?=E5=8E=BB=E5=88=AB=E5=A2=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/BigdataForsaleBatch.groovy | 39 +++++ .../ccbevaluation/BigdataPoiBatch.groovy | 154 ++++++++++++++---- .../script/ccbevaluation/TradeStatLog.groovy | 23 ++- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 39 +++-- 4 files changed, 205 insertions(+), 50 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index d217ba79c..423b23f85 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -13,6 +13,7 @@ import com.fangjia.script.util.FangJiaAreaMapUtils import com.mongodb.BasicDBObject import com.mongodb.DBObject import groovy.sql.Sql +import org.apache.commons.lang.StringUtils import org.bson.types.ObjectId import org.elasticsearch.index.query.BoolQueryBuilder import org.elasticsearch.index.query.QueryBuilder @@ -53,6 +54,7 @@ class BigdataForsaleBatch extends BaseBatch{ FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"]) } + public static final String[] TAGS = {"独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"}; public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", @@ -346,6 +348,10 @@ class BigdataForsaleBatch extends BaseBatch{ if(floor && floor.toInteger()<1){ floor = null } + def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room) + if(vallaFlag){ + continue; + } def rowMap = [:]; if(cat == "rent"){ @@ -410,6 +416,39 @@ class BigdataForsaleBatch extends BaseBatch{ return rets; } + /** + * 是否是别墅 + * @return + */ + public boolean isVilla(area, name, height, room) { + if (area == null) { + return false; + } + + if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) { + return true; + } + + boolean ret = false; + if (height != null && room != null) { + if (height <= 3) { + if (area > 200 && room >= 5) { + ret = true; + } + } + } else if (height == null && room != null) { + if (area > 360 && room >= 6) { + ret = true; + } + } else if (height == null && room == null) { + if (area > 500) { + ret = true; + } + } + + return ret; + } + def getPublicDate(def pDate){ if(pDate>=20181001 && pDate<20200201){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy index 70ec7bde0..f80fa70dc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.ccbevaluation +import com.fangjia.core.RegionData import com.fangjia.core.util.GeoUtils import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner @@ -8,12 +9,13 @@ import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong - +import org.bson.types.ObjectId /** * 建行数据poi同步 */ @@ -31,31 +33,35 @@ class BigdataPoiBatch extends BaseBatch{ def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def citys = CcbCommonUtils.citys; // def path = "D:\\数据\\建行\\房产估值\\poi\\公交.xlsx" - def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1) - def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1) - def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\医院.xlsx", 0,1) - def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1) - def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1) - def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\学校.xlsx", 0,1) - +// def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1) +// def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1) +// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\医院.xlsx", 0,1) +// def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1) +// def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1) +// def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\学校.xlsx", 0,1) + def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\中学.xlsx", 0,1) def rets = [] - def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁") + + def rows1 = createDatas(citys, cityCodeDataMap, hosptialDatas, "学校") rets.addAll(rows1) - def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交") - rets.addAll(rows2) +// def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁") +// rets.addAll(rows1) // - def rows3 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院") - rets.addAll(rows3) +// def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交") +// rets.addAll(rows2) // - def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行") - rets.addAll(rows4) - - def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务") - rets.addAll(rows5) - - def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校") - rets.addAll(rows6) +// def rows3 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院") +// rets.addAll(rows3) +// +// def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行") +// rets.addAll(rows4) +// +// def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务") +// rets.addAll(rows5) +// +// def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校") +// rets.addAll(rows6) def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","来源公司"] @@ -113,7 +119,7 @@ class BigdataPoiBatch extends BaseBatch{ void run() { try{ totalCount.addAndGet(1) - if(citys.contains(d['D'])){ + if(citys.contains(d['A'])){ def ret = addValue(d, cityCodeDataMap, cityDistMap, type) if(ret){ rets << ret @@ -133,24 +139,49 @@ class BigdataPoiBatch extends BaseBatch{ return rets; } - def addValue(d, cityCodeDataMap, cityDistMap, type){ + def getRegionByLocation(lng, lat){ + def jsonObject = GaoDeApiUtils.geocode(lng.toString()+","+lat.toString()) + + if(jsonObject && jsonObject.info=="OK"){ + def compent = jsonObject.regeocode.addressComponent + if(compent){ + def district = compent.district + return district; + } + } + return null + } + + def addValue2(d, cityCodeDataMap, cityDistMap, type){ def ret = [] - def citycode = cityCodeDataMap.get(d['D']) + def citycode = cityCodeDataMap.get(d['E']) ret << citycode - def distCodeMap = cityDistMap.get(d['D']) - if(d['E']){ - def distObj = distCodeMap.get(d['E']) + + def region =getRegionByLocation(d['H'], d['I']) + def distCodeMap = cityDistMap.get(d['E']) + if(region){ + region = RegionData.formatRegion(d['E'], region) + } + if(region){ + def distObj = distCodeMap.get(region) if(distObj){ ret << distObj.district_code }else{ ret << "" - println d['D']+" " + d['E']+" 无对应的区域编码 "+d['F'] + println d['E']+" " + region+" 无对应的区域编码 "+d['F'] } }else{ ret << "" } - ret << d['A'] - ret << d['F'] + + if(d['A']){ + ret << d['A'] + }else{ + ret << new ObjectId() + + } + + ret << d['G'] ret << type if(type=="地铁"){ @@ -192,4 +223,67 @@ class BigdataPoiBatch extends BaseBatch{ return ret; } + + def addValue(d, cityCodeDataMap, cityDistMap, type){ + def ret = [] + def citycode = cityCodeDataMap.get(d['A']) + ret << citycode + def distCodeMap = cityDistMap.get(d['A']) + def region =RegionData.formatRegion(d['A'], d['B']) + + if(region){ + + def distObj = distCodeMap.get(region) + if(distObj){ + ret << distObj.district_code + }else{ + ret << "" + println d['A']+" " + d['B']+" 无对应的区域编码 "+d['F'] + } + }else{ + ret << "" + } + ret << new ObjectId() + ret << d['C'] + ret << type + + if(type=="地铁"){ + ret << "cz125" + }else if(type=="公交"){ + ret << "cz123" + }else if(type=="医院"){ + ret << "hp111" + }else if(type == "银行"){ + ret << "se118" + }else if(type =="丧葬服务"){ + ret << "ob101" + }else if(type =="学校"){ +// if(d['F'].contains("中学")){ + ret << "ed138" +// }else if(d['F'].contains("小学")){ +// ret << "ed114" +// } + } + if(d['I']){ + def lat = d['J'].toDouble() + def lng = d['I'].toDouble() + def loactionMap = GeoUtils.gcjToGps84(lat, lng) + if(loactionMap){ + ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6) + ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6) + }else{ + ret << "" + ret << "" + } + }else{ + ret << "" + ret << "" + } + ret << "" + ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + + return ret; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index e7bd95d9d..c2378636d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -23,8 +23,8 @@ class TradeStatLog { static main(args) { def task = new TradeStatLog() - //task.run() - task.thirdPartyStat() + task.run() +// task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) } @@ -32,33 +32,40 @@ class TradeStatLog { def run(){ def citys = CcbCommonUtils.citys; def rets = [] - rets << ["城市","5月总量","6月总量","贝壳","链家", "我爱我家","房天下","中原"] + rets << ["城市","5月总量","6月总量","7月总量","贝壳","链家", "我爱我家","房天下","中原"] def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"] def startDate = "2021-05-01" def date = "2021-06-01" def endDate = "2021-07-01" + def newDate = "2021-08-01" + for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) def curMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd")) + def nearMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(newDate,"yyyy-MM-dd")) + def ret = [] ret << city if(city == "上海"){ def lastMonthYoudaCount = getYouDaData(DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) def curMonthYoudaCount = getYouDaData(DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd")) + def nearMonthYoudaCount = getYouDaData(DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(newDate,"yyyy-MM-dd")) def a = lastMonthCount.toInteger() + lastMonthYoudaCount.toInteger() def b = curMonthCount.toInteger() + curMonthYoudaCount.toInteger() + def c = nearMonthCount.toInteger() + nearMonthYoudaCount.toInteger() ret << a ret << b - def c = (b-a)*100/a - def pre = DataHandleUtils.getDoubleRound(2, c) + ret << c + def d = (c-b)*100/a + def pre = DataHandleUtils.getDoubleRound(2, d) ret << pre }else{ ret << lastMonthCount ret << curMonthCount - + ret << nearMonthCount if(lastMonthCount){ - def a = (curMonthCount-lastMonthCount)*100/lastMonthCount + def a = (nearMonthCount-curMonthCount)*100/curMonthCount def pre = DataHandleUtils.getDoubleRound(2, a) ret << pre }else{ @@ -70,8 +77,10 @@ class TradeStatLog { for(def source : sources){ def sourceLastMonthCount = findTradesCount(city, source, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) def sourceCurMonthCount = findTradesCount(city, source, DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd")) + def sourceNearMonthCount = findTradesCount(city, source, DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(newDate,"yyyy-MM-dd")) ret << sourceLastMonthCount ret << sourceCurMonthCount + ret << sourceNearMonthCount } rets << ret println ret diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index bbd42ab13..2b8980257 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -4,6 +4,7 @@ import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils import com.mongodb.BasicDBObject import org.apache.solr.common.util.Hash +import org.bson.types.ObjectId /** * 福州农商行 @@ -16,7 +17,7 @@ class SynchDistrictPrice { def districtDetailColl = CrawalDBUtil.getDistrictDetailColl(); // def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房", // "anjuke.com":"安居客","cityhouse.cn":"城市房产"] - def sourceMap = ["zhuge.com":"诸葛找房"] + def sourceMap = ["lianjia.com":"链家"] static main(args) { def task = new SynchDistrictPrice() task.run() @@ -27,11 +28,13 @@ class SynchDistrictPrice { def citys = CommonUtils.findAllCity() sourceMap.each { k, v -> - for(def city : citys){ - def rets = findCrawlDistrictDetails(city, k, v) - println city +" "+rets.size() - synchData(city, rets, v) + for(def city : citys){ + if(!["上海","北京","海口"].contains(city)){ + def rets = findCrawlDistrictDetails(city, k, v) + println city +" "+rets.size() + synchData(city, rets, v) + } } } @@ -77,6 +80,7 @@ class SynchDistrictPrice { q.put("city", city) def count = thirdPartyColl.count(q) def addIndex = 0 + def updateIndex = 0 if(count==0){ try{ thirdPartyColl.insert(rets) @@ -86,13 +90,25 @@ class SynchDistrictPrice { println "["+source +"-"+city + "] add:"+ rets.size()+" 原抓取:"+count }else{ for(def r : rets){ - def flag = findIsExists(r.city, r.region, r.name, SHORT_DATE, source) - if(!flag){ + def obj= findIsExists(r.city, r.region, r.name, SHORT_DATE, source) + if(obj){ + thirdPartyColl.remove(new BasicDBObject('_id',obj._id)) addIndex++ thirdPartyColl.save(r) } +// if(!obj){ +// addIndex++ +// thirdPartyColl.save(r) +// }else{ +// if(r.avg_price && !obj.avg_price){ +// def udata = new BasicDBObject() +// udata.put("avg_price", r.avg_price) +// thirdPartyColl.update(new BasicDBObject('_id',obj._id),new BasicDBObject('$set',udata)) +// updateIndex++ +// } +// } } - println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count + println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count +" 修改:"+updateIndex } @@ -105,10 +121,7 @@ class SynchDistrictPrice { q.put("city", city) q.put("region", region) q.put("name", name) - def count = thirdPartyColl.count(q) - if(count){ - return true - } - return false + def obj = thirdPartyColl.findOne(q) + return obj } }