+39
@@ -13,6 +13,7 @@ import com.fangjia.script.util.FangJiaAreaMapUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import groovy.sql.Sql
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
@@ -53,6 +54,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
|
||||
}
|
||||
|
||||
public static final String[] TAGS = {"独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"};
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
@@ -346,6 +348,10 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
if(floor && floor.toInteger()<1){
|
||||
floor = null
|
||||
}
|
||||
def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
|
||||
if(vallaFlag){
|
||||
continue;
|
||||
}
|
||||
|
||||
def rowMap = [:];
|
||||
if(cat == "rent"){
|
||||
@@ -410,6 +416,39 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
return rets;
|
||||
}
|
||||
|
||||
/**
|
||||
* 是否是别墅
|
||||
* @return
|
||||
*/
|
||||
public boolean isVilla(area, name, height, room) {
|
||||
if (area == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) {
|
||||
return true;
|
||||
}
|
||||
|
||||
boolean ret = false;
|
||||
if (height != null && room != null) {
|
||||
if (height <= 3) {
|
||||
if (area > 200 && room >= 5) {
|
||||
ret = true;
|
||||
}
|
||||
}
|
||||
} else if (height == null && room != null) {
|
||||
if (area > 360 && room >= 6) {
|
||||
ret = true;
|
||||
}
|
||||
} else if (height == null && room == null) {
|
||||
if (area > 500) {
|
||||
ret = true;
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
def getPublicDate(def pDate){
|
||||
|
||||
if(pDate>=20181001 && pDate<20200201){
|
||||
|
||||
+124
-30
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
@@ -8,12 +9,13 @@ import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
import org.bson.types.ObjectId
|
||||
/**
|
||||
* 建行数据poi同步
|
||||
*/
|
||||
@@ -31,31 +33,35 @@ class BigdataPoiBatch extends BaseBatch{
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def citys = CcbCommonUtils.citys;
|
||||
// def path = "D:\\数据\\建行\\房产估值\\poi\\公交.xlsx"
|
||||
def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1)
|
||||
def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1)
|
||||
def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\医院.xlsx", 0,1)
|
||||
def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1)
|
||||
def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1)
|
||||
def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\学校.xlsx", 0,1)
|
||||
|
||||
// def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1)
|
||||
// def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1)
|
||||
// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\医院.xlsx", 0,1)
|
||||
// def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1)
|
||||
// def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1)
|
||||
// def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\学校.xlsx", 0,1)
|
||||
def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\中学.xlsx", 0,1)
|
||||
def rets = []
|
||||
def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁")
|
||||
|
||||
def rows1 = createDatas(citys, cityCodeDataMap, hosptialDatas, "学校")
|
||||
rets.addAll(rows1)
|
||||
|
||||
def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交")
|
||||
rets.addAll(rows2)
|
||||
// def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁")
|
||||
// rets.addAll(rows1)
|
||||
//
|
||||
def rows3 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院")
|
||||
rets.addAll(rows3)
|
||||
// def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交")
|
||||
// rets.addAll(rows2)
|
||||
//
|
||||
def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行")
|
||||
rets.addAll(rows4)
|
||||
|
||||
def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务")
|
||||
rets.addAll(rows5)
|
||||
|
||||
def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校")
|
||||
rets.addAll(rows6)
|
||||
// def rows3 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院")
|
||||
// rets.addAll(rows3)
|
||||
//
|
||||
// def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行")
|
||||
// rets.addAll(rows4)
|
||||
//
|
||||
// def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务")
|
||||
// rets.addAll(rows5)
|
||||
//
|
||||
// def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校")
|
||||
// rets.addAll(rows6)
|
||||
|
||||
def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","来源公司"]
|
||||
|
||||
@@ -113,7 +119,7 @@ class BigdataPoiBatch extends BaseBatch{
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
if(citys.contains(d['D'])){
|
||||
if(citys.contains(d['A'])){
|
||||
def ret = addValue(d, cityCodeDataMap, cityDistMap, type)
|
||||
if(ret){
|
||||
rets << ret
|
||||
@@ -133,24 +139,49 @@ class BigdataPoiBatch extends BaseBatch{
|
||||
return rets;
|
||||
}
|
||||
|
||||
def addValue(d, cityCodeDataMap, cityDistMap, type){
|
||||
def getRegionByLocation(lng, lat){
|
||||
def jsonObject = GaoDeApiUtils.geocode(lng.toString()+","+lat.toString())
|
||||
|
||||
if(jsonObject && jsonObject.info=="OK"){
|
||||
def compent = jsonObject.regeocode.addressComponent
|
||||
if(compent){
|
||||
def district = compent.district
|
||||
return district;
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def addValue2(d, cityCodeDataMap, cityDistMap, type){
|
||||
def ret = []
|
||||
def citycode = cityCodeDataMap.get(d['D'])
|
||||
def citycode = cityCodeDataMap.get(d['E'])
|
||||
ret << citycode
|
||||
def distCodeMap = cityDistMap.get(d['D'])
|
||||
if(d['E']){
|
||||
def distObj = distCodeMap.get(d['E'])
|
||||
|
||||
def region =getRegionByLocation(d['H'], d['I'])
|
||||
def distCodeMap = cityDistMap.get(d['E'])
|
||||
if(region){
|
||||
region = RegionData.formatRegion(d['E'], region)
|
||||
}
|
||||
if(region){
|
||||
def distObj = distCodeMap.get(region)
|
||||
if(distObj){
|
||||
ret << distObj.district_code
|
||||
}else{
|
||||
ret << ""
|
||||
println d['D']+" " + d['E']+" 无对应的区域编码 "+d['F']
|
||||
println d['E']+" " + region+" 无对应的区域编码 "+d['F']
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
ret << d['A']
|
||||
ret << d['F']
|
||||
|
||||
if(d['A']){
|
||||
ret << d['A']
|
||||
}else{
|
||||
ret << new ObjectId()
|
||||
|
||||
}
|
||||
|
||||
ret << d['G']
|
||||
ret << type
|
||||
|
||||
if(type=="地铁"){
|
||||
@@ -192,4 +223,67 @@ class BigdataPoiBatch extends BaseBatch{
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
def addValue(d, cityCodeDataMap, cityDistMap, type){
|
||||
def ret = []
|
||||
def citycode = cityCodeDataMap.get(d['A'])
|
||||
ret << citycode
|
||||
def distCodeMap = cityDistMap.get(d['A'])
|
||||
def region =RegionData.formatRegion(d['A'], d['B'])
|
||||
|
||||
if(region){
|
||||
|
||||
def distObj = distCodeMap.get(region)
|
||||
if(distObj){
|
||||
ret << distObj.district_code
|
||||
}else{
|
||||
ret << ""
|
||||
println d['A']+" " + d['B']+" 无对应的区域编码 "+d['F']
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
ret << new ObjectId()
|
||||
ret << d['C']
|
||||
ret << type
|
||||
|
||||
if(type=="地铁"){
|
||||
ret << "cz125"
|
||||
}else if(type=="公交"){
|
||||
ret << "cz123"
|
||||
}else if(type=="医院"){
|
||||
ret << "hp111"
|
||||
}else if(type == "银行"){
|
||||
ret << "se118"
|
||||
}else if(type =="丧葬服务"){
|
||||
ret << "ob101"
|
||||
}else if(type =="学校"){
|
||||
// if(d['F'].contains("中学")){
|
||||
ret << "ed138"
|
||||
// }else if(d['F'].contains("小学")){
|
||||
// ret << "ed114"
|
||||
// }
|
||||
}
|
||||
if(d['I']){
|
||||
def lat = d['J'].toDouble()
|
||||
def lng = d['I'].toDouble()
|
||||
def loactionMap = GeoUtils.gcjToGps84(lat, lng)
|
||||
if(loactionMap){
|
||||
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
|
||||
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
+16
-7
@@ -23,8 +23,8 @@ class TradeStatLog {
|
||||
|
||||
static main(args) {
|
||||
def task = new TradeStatLog()
|
||||
//task.run()
|
||||
task.thirdPartyStat()
|
||||
task.run()
|
||||
// task.thirdPartyStat()
|
||||
// println Calendar.getInstance().get(Calendar.YEAR)
|
||||
|
||||
}
|
||||
@@ -32,33 +32,40 @@ class TradeStatLog {
|
||||
def run(){
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rets = []
|
||||
rets << ["城市","5月总量","6月总量","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
rets << ["城市","5月总量","6月总量","7月总量","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
|
||||
def startDate = "2021-05-01"
|
||||
def date = "2021-06-01"
|
||||
def endDate = "2021-07-01"
|
||||
def newDate = "2021-08-01"
|
||||
|
||||
for(def city : citys){
|
||||
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
def curMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
|
||||
def nearMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(newDate,"yyyy-MM-dd"))
|
||||
|
||||
def ret = []
|
||||
ret << city
|
||||
|
||||
if(city == "上海"){
|
||||
def lastMonthYoudaCount = getYouDaData(DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
def curMonthYoudaCount = getYouDaData(DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
|
||||
def nearMonthYoudaCount = getYouDaData(DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(newDate,"yyyy-MM-dd"))
|
||||
def a = lastMonthCount.toInteger() + lastMonthYoudaCount.toInteger()
|
||||
def b = curMonthCount.toInteger() + curMonthYoudaCount.toInteger()
|
||||
def c = nearMonthCount.toInteger() + nearMonthYoudaCount.toInteger()
|
||||
ret << a
|
||||
ret << b
|
||||
def c = (b-a)*100/a
|
||||
def pre = DataHandleUtils.getDoubleRound(2, c)
|
||||
ret << c
|
||||
def d = (c-b)*100/a
|
||||
def pre = DataHandleUtils.getDoubleRound(2, d)
|
||||
ret << pre
|
||||
}else{
|
||||
ret << lastMonthCount
|
||||
ret << curMonthCount
|
||||
|
||||
ret << nearMonthCount
|
||||
if(lastMonthCount){
|
||||
def a = (curMonthCount-lastMonthCount)*100/lastMonthCount
|
||||
def a = (nearMonthCount-curMonthCount)*100/curMonthCount
|
||||
def pre = DataHandleUtils.getDoubleRound(2, a)
|
||||
ret << pre
|
||||
}else{
|
||||
@@ -70,8 +77,10 @@ class TradeStatLog {
|
||||
for(def source : sources){
|
||||
def sourceLastMonthCount = findTradesCount(city, source, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
def sourceCurMonthCount = findTradesCount(city, source, DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
|
||||
def sourceNearMonthCount = findTradesCount(city, source, DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(newDate,"yyyy-MM-dd"))
|
||||
ret << sourceLastMonthCount
|
||||
ret << sourceCurMonthCount
|
||||
ret << sourceNearMonthCount
|
||||
}
|
||||
rets << ret
|
||||
println ret
|
||||
|
||||
+26
-13
@@ -4,6 +4,7 @@ import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.apache.solr.common.util.Hash
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
/**
|
||||
* 福州农商行
|
||||
@@ -16,7 +17,7 @@ class SynchDistrictPrice {
|
||||
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
|
||||
// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
|
||||
// "anjuke.com":"安居客","cityhouse.cn":"城市房产"]
|
||||
def sourceMap = ["zhuge.com":"诸葛找房"]
|
||||
def sourceMap = ["lianjia.com":"链家"]
|
||||
static main(args) {
|
||||
def task = new SynchDistrictPrice()
|
||||
task.run()
|
||||
@@ -27,11 +28,13 @@ class SynchDistrictPrice {
|
||||
def citys = CommonUtils.findAllCity()
|
||||
|
||||
sourceMap.each { k, v ->
|
||||
for(def city : citys){
|
||||
def rets = findCrawlDistrictDetails(city, k, v)
|
||||
println city +" "+rets.size()
|
||||
synchData(city, rets, v)
|
||||
|
||||
for(def city : citys){
|
||||
if(!["上海","北京","海口"].contains(city)){
|
||||
def rets = findCrawlDistrictDetails(city, k, v)
|
||||
println city +" "+rets.size()
|
||||
synchData(city, rets, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -77,6 +80,7 @@ class SynchDistrictPrice {
|
||||
q.put("city", city)
|
||||
def count = thirdPartyColl.count(q)
|
||||
def addIndex = 0
|
||||
def updateIndex = 0
|
||||
if(count==0){
|
||||
try{
|
||||
thirdPartyColl.insert(rets)
|
||||
@@ -86,13 +90,25 @@ class SynchDistrictPrice {
|
||||
println "["+source +"-"+city + "] add:"+ rets.size()+" 原抓取:"+count
|
||||
}else{
|
||||
for(def r : rets){
|
||||
def flag = findIsExists(r.city, r.region, r.name, SHORT_DATE, source)
|
||||
if(!flag){
|
||||
def obj= findIsExists(r.city, r.region, r.name, SHORT_DATE, source)
|
||||
if(obj){
|
||||
thirdPartyColl.remove(new BasicDBObject('_id',obj._id))
|
||||
addIndex++
|
||||
thirdPartyColl.save(r)
|
||||
}
|
||||
// if(!obj){
|
||||
// addIndex++
|
||||
// thirdPartyColl.save(r)
|
||||
// }else{
|
||||
// if(r.avg_price && !obj.avg_price){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("avg_price", r.avg_price)
|
||||
// thirdPartyColl.update(new BasicDBObject('_id',obj._id),new BasicDBObject('$set',udata))
|
||||
// updateIndex++
|
||||
// }
|
||||
// }
|
||||
}
|
||||
println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count
|
||||
println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count +" 修改:"+updateIndex
|
||||
}
|
||||
|
||||
|
||||
@@ -105,10 +121,7 @@ class SynchDistrictPrice {
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
def count = thirdPartyColl.count(q)
|
||||
if(count){
|
||||
return true
|
||||
}
|
||||
return false
|
||||
def obj = thirdPartyColl.findOne(q)
|
||||
return obj
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user