Merge remote-tracking branch 'origin/master'
This commit is contained in:
+1
-1
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
|
||||
/**
|
||||
* 深圳公众号成交数据录入
|
||||
*/
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年7月深圳二手房成交.xlsx", 0, 1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年9月深圳二手房成交.xlsx", 0, 1)
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
|
||||
+163
-7
@@ -1,11 +1,18 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 将国信达及成交对标数据加入到第三方均价表,做为均价参考
|
||||
*/
|
||||
@@ -13,19 +20,30 @@ class AddThirdPriceSample {
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
public static final DBCollection NO_PRICE_DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "deal_price_all", "mongo.db.fangjia.com")
|
||||
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
static main(args) {
|
||||
def task = new AddThirdPriceSample()
|
||||
task.run()
|
||||
// task.run() //添加第三方 fccs->保存的是国信达的参考价格 csjj->保存的是国信达的成交数据
|
||||
task.addTrade() //添加成交
|
||||
|
||||
// task.matchOldData()
|
||||
// task.noInCcbCity()
|
||||
|
||||
}
|
||||
def run(){
|
||||
for(def i=1;i<19;i++){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\级联待优化小区集合_7月.xlsx",{d->
|
||||
if(d['A'] && d['C']){
|
||||
for(def i=0;i<1;i++){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\国信达指导价录入-1024.xlsx",{d->
|
||||
if(d['A'] && d['B']){
|
||||
def dist = CommonDatautils.findDistrictById(d['A'].toString().trim())
|
||||
if(dist){
|
||||
def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202407)
|
||||
def udata = new BasicDBObject("csjj_esf_price", d['C'].toString().toDouble().toInteger())
|
||||
|
||||
def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202409)
|
||||
// def udata = new BasicDBObject("csjj_esf_price", d['C'].toString().toDouble().toInteger())
|
||||
def udata = new BasicDBObject("fccs_esf_price", d['B'].toString().toDouble().toInteger())
|
||||
AVERAGE_STATISTICS.update(q,new BasicDBObject('$set', udata),false, false)
|
||||
}else{
|
||||
println d['A'] +" 没查到"
|
||||
@@ -37,4 +55,142 @@ class AddThirdPriceSample {
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def addTrade(){
|
||||
// def citys = ["武汉","南京","杭州","长沙","郑州","合肥","宁波","南通","嘉兴","珠海","泉州","绍兴","湖州","镇江"]
|
||||
def rets = findTrades()
|
||||
DEAL_PRICE_COLL.insert(rets)
|
||||
}
|
||||
|
||||
def findTrades(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("ccb_price_remark","建行无此城市成交")
|
||||
// q.put("city", new BasicDBObject('$in', citys))
|
||||
// q.put("flag", 1)
|
||||
def dbcursor = NO_PRICE_DEAL_PRICE_COLL.find(q)
|
||||
def rets = []
|
||||
def index = 0
|
||||
for(def db : dbcursor){
|
||||
index++
|
||||
def udata = new BasicDBObject()
|
||||
|
||||
udata.put("_id",db._id)
|
||||
udata.put("url",db.url)
|
||||
udata.put("area",db.area.toString().toDouble())
|
||||
udata.put("avg_price",db.avg_price.toString().toDouble().toInteger())
|
||||
udata.put("build_structure",db.build_structure)
|
||||
udata.put("build_type",db.build_type)
|
||||
udata.put("build_year",db.build_year)
|
||||
udata.put("city",db.city)
|
||||
udata.put("direction", db.direction)
|
||||
udata.put("district_name",db.district_name)
|
||||
udata.put("elevator",db.elevator)
|
||||
udata.put("fitment",db.fitment)
|
||||
udata.put("floor_info",db.floor_info)
|
||||
if(db.hall){
|
||||
udata.put("hall",db.hall.toString().toInteger())
|
||||
}
|
||||
|
||||
if(db.height){
|
||||
udata.put("height",db.height.toString().toInteger())
|
||||
}
|
||||
|
||||
udata.put("housing_age",db.housing_age)
|
||||
udata.put("lift",db.lift.toString().toInteger())
|
||||
udata.put("lift_household",db.lift_household.toString().toInteger())
|
||||
udata.put("property_ownership",db.property_ownership)
|
||||
udata.put("region",db.region)
|
||||
udata.put("room", db.room.toString().toInteger())
|
||||
udata.put("room_structure",db.room_structure)
|
||||
udata.put("source",db.source)
|
||||
udata.put("total_price",db.total_price.toString().toDouble().toInteger())
|
||||
udata.put("trade_date",db.trade_date)
|
||||
udata.put("trade_property",db.trade_property)
|
||||
udata.put("house_using",db.house_using)
|
||||
udata.put("heating_type",db.heating_type)
|
||||
udata.put("community_id",db.community_id)
|
||||
udata.put("create_date",db.create_time)
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("address",db.address)
|
||||
udata.put("match_city",db.match_city)
|
||||
udata.put("match_region",db.match_region)
|
||||
udata.put("match_name",db.match_name)
|
||||
udata.put("match_id",db.match_id)
|
||||
udata.put("price_flag", "国信达成交-推导")
|
||||
|
||||
// DEAL_PRICE_COLL.save(udata)
|
||||
println index
|
||||
rets << udata
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
def matchOldData(){
|
||||
def q = new BasicDBObject("source","贝壳找房").append("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2024-09-01","yyyy-MM-dd")))
|
||||
def list = DEAL_PRICE_COLL.find(q).toList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(list.size())
|
||||
for(def i=0;i<list.size();i++){
|
||||
final def d = list[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def query = new BasicDBObject("url", d.url)
|
||||
def obj = NO_PRICE_DEAL_PRICE_COLL.findOne(query)
|
||||
if(obj){
|
||||
def update = new BasicDBObject()
|
||||
update.put("price_remark", "有价格")
|
||||
NO_PRICE_DEAL_PRICE_COLL.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set', update), false,false)
|
||||
}
|
||||
println totalCount.get()
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
def noInCcbCity(){
|
||||
def citys = ["大理","苏州","无锡","通辽","绵阳","吉安","潍坊","襄阳","济宁","张家口","泸州","乐山","赣州","九江","眉山","桂林","雅安","广州","景德镇","上海","运城","内江","金华","濮阳","丹东","淄博","新乡","廊坊","烟台","邯郸","萍乡","宜宾","咸阳","许昌","德阳","银川","洛阳","天津","湛江","长沙","深圳","湘西","信阳","成都","威海","福州","上饶","常州","宜春","天水","平顶山","常德","吉林","盐城","杭州","赤峰","南京","周口","济南","株洲","大连","徐州","泰安","太原","凉山","兰州","承德","广元","台州","南昌","乌鲁木齐","海口","北京","淮安","清远","临沂","柳州","衢州","攀枝花","呼和浩特","昆明","岳阳","衡阳","温州","石家庄","连云港","保定","阜阳","遂宁","菏泽","青岛","唐山","开封","漳州","贵阳","驻马店","宝鸡","达州","汉中","南充","遵义","包头","南宁","沈阳","重庆","芜湖","厦门","大同"]
|
||||
def q = new BasicDBObject()
|
||||
q.put("price_remark",new BasicDBObject('$ne', "有价格"))
|
||||
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2024-09-01","yyyy-MM-dd")))
|
||||
q.put("avg_price",new BasicDBObject('$gt', 0))
|
||||
q.put("match_id",new BasicDBObject('$exists', true))
|
||||
def list = NO_PRICE_DEAL_PRICE_COLL.find(q).toList()
|
||||
println "共:"+list.size()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(list.size())
|
||||
for(def i=0;i<list.size();i++){
|
||||
final def d = list[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
|
||||
if(!citys.contains(d.city)){
|
||||
def update = new BasicDBObject()
|
||||
update.put("ccb_price_remark", "建行无此城市成交")
|
||||
NO_PRICE_DEAL_PRICE_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set', update), false,false)
|
||||
}
|
||||
println totalCount.get()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+5
-5
@@ -71,11 +71,11 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
|
||||
println "curDate:"+curDate
|
||||
def allCitys = CommonDatautils.findAllCity()
|
||||
for(def city : allCitys){
|
||||
addDealPriceMatch(city);
|
||||
}
|
||||
updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
|
||||
// def allCitys = CommonDatautils.findAllCity()
|
||||
// for(def city : allCitys){
|
||||
// addDealPriceMatch(city);
|
||||
// }
|
||||
// updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
|
||||
|
||||
|
||||
def rows = []
|
||||
|
||||
+1
-1
@@ -40,7 +40,7 @@ class CcbCommonUtils {
|
||||
|
||||
public static final DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
// public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "deal_price_all", "mongo.db.fangjia.com")
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
|
||||
public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017)
|
||||
|
||||
+63
-26
@@ -5,9 +5,14 @@ import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import groovy.sql.Sql
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
|
||||
/**
|
||||
* 建行成交数据表优化
|
||||
*/
|
||||
@@ -24,6 +29,8 @@ class DealPriceDataProcess {
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
static main(args) {
|
||||
def task = new DealPriceDataProcess()
|
||||
// task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据
|
||||
@@ -394,43 +401,73 @@ class DealPriceDataProcess {
|
||||
def rows = []
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def set = new HashSet()
|
||||
def index = 0
|
||||
for(def city : citys) {
|
||||
index ++
|
||||
if(index<145){
|
||||
continue
|
||||
}
|
||||
println index +" "+city
|
||||
def cityid = cityCodeDatas.get(city)
|
||||
def rets = findPeGteThirdParty2(cityid, DateUtils.formatDateByPattern(new Date(),"yyyyMM"))//当前月日期
|
||||
println city +" "+rets.size()
|
||||
if(rets){
|
||||
set.add(city)
|
||||
}
|
||||
for(def m : rets){
|
||||
|
||||
// def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
|
||||
// sqlInstance236.executeUpdate(sql)
|
||||
def querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
|
||||
def total = 0
|
||||
pgsqlInstance.eachRow(querySql, { r ->
|
||||
total = r.count
|
||||
})
|
||||
if(total>0){
|
||||
println city +" "+m.commid+" (按价格)删除:"+total
|
||||
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
|
||||
pgsqlInstance.executeUpdate(sql)
|
||||
}else{
|
||||
|
||||
querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} "
|
||||
pgsqlInstance.eachRow(querySql, { r ->
|
||||
total = r.count
|
||||
})
|
||||
if(total>0){
|
||||
println city +" "+m.commid+" (按小区)删除:"+total
|
||||
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} "
|
||||
pgsqlInstance.executeUpdate(sql)
|
||||
}
|
||||
}
|
||||
}
|
||||
processDetele(city, cityid, month, rets)
|
||||
}
|
||||
println set.toString()
|
||||
}
|
||||
|
||||
|
||||
def processDetele(city, cityid, month, rets){
|
||||
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
|
||||
for(def i=0;i<rets.size();i++){
|
||||
final def m = rets[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
// def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
|
||||
// sqlInstance236.executeUpdate(sql)
|
||||
def querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
|
||||
def total = 0
|
||||
pgsqlInstance.eachRow(querySql, { r ->
|
||||
total = r.count
|
||||
})
|
||||
if(total>0){
|
||||
println city +" "+m.commid+" (按价格)删除:"+total
|
||||
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
|
||||
pgsqlInstance.executeUpdate(sql)
|
||||
}else{
|
||||
|
||||
querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} "
|
||||
pgsqlInstance.eachRow(querySql, { r ->
|
||||
total = r.count
|
||||
})
|
||||
if(total>0){
|
||||
println city +" "+m.commid+" (按小区)删除:"+total
|
||||
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} "
|
||||
pgsqlInstance.executeUpdate(sql)
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
latch.await()
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def findThirdCount(sqlInstance236, sql){
|
||||
def count = 0
|
||||
sqlInstance236.eachRow(sql,{r->
|
||||
|
||||
+55
-52
@@ -631,36 +631,36 @@ def findAllData(sqlInstance236, citycode){
|
||||
//},0)
|
||||
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\添加别名.xlsx",{d->
|
||||
if(d['A'] && d['B']){
|
||||
def dbo = CommonDatautils.findDistrictById(d['A'])
|
||||
if(dbo){
|
||||
def alias = null
|
||||
if(!dbo.alias){
|
||||
alias = []
|
||||
}else{
|
||||
alias = dbo.alias
|
||||
}
|
||||
if(!alias.contains(d['B'])){
|
||||
alias << d['B']
|
||||
}
|
||||
if(alias){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("alias", alias)
|
||||
udata.put("m_date",new Date())
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", dbo._id)
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
println d['A']+" "+"别名已存在"
|
||||
}
|
||||
|
||||
}else{
|
||||
println dbo._id.toString() +" 没找到"
|
||||
}
|
||||
}
|
||||
})
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\添加别名.xlsx",{d->
|
||||
// if(d['A'] && d['B']){
|
||||
// def dbo = CommonDatautils.findDistrictById(d['A'])
|
||||
// if(dbo){
|
||||
// def alias = null
|
||||
// if(!dbo.alias){
|
||||
// alias = []
|
||||
// }else{
|
||||
// alias = dbo.alias
|
||||
// }
|
||||
// if(!alias.contains(d['B'])){
|
||||
// alias << d['B']
|
||||
// }
|
||||
// if(alias){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("alias", alias)
|
||||
// udata.put("m_date",new Date())
|
||||
//
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("_id", dbo._id)
|
||||
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
// }else{
|
||||
// println d['A']+" "+"别名已存在"
|
||||
// }
|
||||
//
|
||||
// }else{
|
||||
// println dbo._id.toString() +" 没找到"
|
||||
// }
|
||||
// }
|
||||
//})
|
||||
|
||||
//def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]
|
||||
//for(def c : citys){
|
||||
@@ -1234,15 +1234,7 @@ def findThirdCommCount(cityid, sqlInstance){
|
||||
//}
|
||||
|
||||
//
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\无价格小区(3)2.23.xlsx"
|
||||
//ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
// if(dist){
|
||||
// d['G'] = dist.city
|
||||
// d['H'] = dist.region
|
||||
// d['I'] = dist.name
|
||||
// }
|
||||
//})
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -1331,10 +1323,10 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
|
||||
|
||||
|
||||
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240806--国信达价格录入.xlsx"
|
||||
//
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\6个城市待优化_9月xlsx.xlsx"
|
||||
////
|
||||
//ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
// if(d['J']){
|
||||
// if(d['G']){
|
||||
//// def region = RegionData.formatRegion(d['A'], d['J'])
|
||||
//// if(region){
|
||||
//// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district")
|
||||
@@ -1344,23 +1336,34 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
|
||||
//// d['L'] = dist.status-dist.name_cat
|
||||
//// }
|
||||
//// }
|
||||
//// if(d['J'] instanceof String){
|
||||
// def dist = CommonDatautils.findDistrictById(d['J'])
|
||||
// if(d['G'] instanceof String){
|
||||
// def dist = CommonDatautils.findDistrictById(d['G'])
|
||||
// if(dist){
|
||||
// d['A'] = dist.city
|
||||
// d['B'] = dist.region
|
||||
// d['C'] = dist.name
|
||||
// d['K'] = dist.estate_type2
|
||||
// d['L'] = dist.status
|
||||
// d['M'] = dist.name_cat
|
||||
//// d['A'] = dist.city
|
||||
//// d['B'] = dist.region
|
||||
//// d['C'] = dist.name
|
||||
//// d['K'] = dist.estate_type2
|
||||
//// d['L'] = dist.status
|
||||
//// d['M'] = dist.name_cat
|
||||
//
|
||||
//// d['H'] = dist.name
|
||||
// d['H'] = dist.name
|
||||
// }
|
||||
//// }
|
||||
// }
|
||||
// }
|
||||
//},0)
|
||||
|
||||
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20241025人工指导价.xlsx"
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
def dist = CommonDatautils.findDistrictById(d['J'])
|
||||
if(dist){
|
||||
d['A'] = dist.city
|
||||
d['B'] = dist.region
|
||||
d['C'] = dist.name
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
//def file = "D:\\数据\\建行\\房产估值\\指导价\\5城价格翻新6-21.xlsx"
|
||||
//ExcelUtils.multiThreadingModify(file, { d ->
|
||||
// if(d['A']){
|
||||
|
||||
+18
-3
@@ -47,7 +47,8 @@ class TradeStatLog {
|
||||
// task.run()//统计最近3个月抓取的成交数据量
|
||||
// task.exportTrade()//导出未入库成交
|
||||
|
||||
task.exportTrade2()//导出未入库成交
|
||||
// task.exportTrade2()//导出未入库成交
|
||||
task.updateDealPriceRegion()//修改成交区域数据
|
||||
// task.thirdPartyStat()
|
||||
// println Calendar.getInstance().get(Calendar.YEAR)
|
||||
// task.averageStat()
|
||||
@@ -597,7 +598,7 @@ class TradeStatLog {
|
||||
// def endDate = "2024-07-01"
|
||||
def ids = findAllDealIds();
|
||||
println ids.size()
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\其它\\2024年6月成交数据.xlsx",{d->
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\建行成交\\廊坊.xlsx",{d->
|
||||
if(ids.contains(d['A'])){
|
||||
d['V'] = "供"
|
||||
}
|
||||
@@ -674,7 +675,7 @@ class TradeStatLog {
|
||||
}
|
||||
|
||||
def findAllDealIds(){
|
||||
def sql = "select gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where p9_start_date='2024-07-28'"
|
||||
def sql = "select gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where urbn_nm='廊坊'"
|
||||
def ids = []
|
||||
pssqlInstance.eachRow(sql,{r->
|
||||
if(!ids.contains(r.gujia_bds_mdl_id)){
|
||||
@@ -801,6 +802,20 @@ class TradeStatLog {
|
||||
},0)
|
||||
}
|
||||
|
||||
def updateDealPriceRegion(){
|
||||
ExcelUtils.modify("D:\\数据\\建行\\建行成交\\成交待修改区域数据.xlsx",{d->
|
||||
if(d['A'] && d['G']){
|
||||
def region = RegionData.formatRegion(d['D'], d['G'])
|
||||
if(region){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("region", region)
|
||||
DEAL_PRICE_COLL.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata))
|
||||
}
|
||||
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def findTradeById(id){
|
||||
def q = new BasicDBObject()
|
||||
|
||||
+59
-38
@@ -24,6 +24,13 @@ class AddressCut extends BaseBatch{
|
||||
|
||||
final DBCollection thirdPartyPriceColl = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018)
|
||||
|
||||
final DBCollection thirdPartyPriceBkColl = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price_history_bk_new", "192.168.0.235", 27018)
|
||||
|
||||
|
||||
final DBCollection dataProcessColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process", "192.168.0.235", 27018)
|
||||
|
||||
final DBCollection dataProcessBkColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process_history_bk", "192.168.0.235", 27018)
|
||||
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
|
||||
static {
|
||||
@@ -39,50 +46,66 @@ class AddressCut extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
<<<<<<< HEAD
|
||||
// sameAddr()
|
||||
seaweedStreetUpdate()
|
||||
def dates = [202408]
|
||||
for(def date : dates){
|
||||
// synch(date)
|
||||
synchDataProcess(date)
|
||||
println date + "同步完成! "
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def seaweedStreetUpdate(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区地址拆分\\已核实-路2.xlsx",{d->
|
||||
if(d['A']){
|
||||
def dbo = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
|
||||
if(dbo){
|
||||
def udate = new BasicDBObject()
|
||||
def street = null
|
||||
def address = ""
|
||||
if(d['E']){
|
||||
street = d['E']
|
||||
udate.put("street", street)
|
||||
address = street
|
||||
}
|
||||
def streetNum = null
|
||||
if(d['F']){
|
||||
streetNum = d['F']
|
||||
udate.put("street_number", streetNum)
|
||||
address = address + streetNum
|
||||
}
|
||||
|
||||
if(dbo.address!=address){
|
||||
udate.put("address", address)
|
||||
}
|
||||
if(udate){
|
||||
udate.put("m_date", new Date())
|
||||
DBUtils.seaweed.update(new BasicDBObject('_id', dbo.get("_id")), new BasicDBObject('$set', udate))
|
||||
}
|
||||
|
||||
}
|
||||
def synch(date){
|
||||
def q = new BasicDBObject()
|
||||
q.put("s_date", date)
|
||||
def dbcursor = thirdPartyPriceColl.find(q).batchSize(5000)
|
||||
def index = 0
|
||||
def rets = []
|
||||
def num = 0
|
||||
for(def d : dbcursor){
|
||||
num++
|
||||
index++
|
||||
rets << d
|
||||
if(index%10000==0){
|
||||
thirdPartyPriceBkColl.insert(rets)
|
||||
println(num)
|
||||
rets.clear()
|
||||
}
|
||||
|
||||
})
|
||||
=======
|
||||
// sameAddr() //相同地址小区
|
||||
// roadMatch()
|
||||
addAnjukeRoad()
|
||||
|
||||
}
|
||||
if(rets.size()>0){
|
||||
thirdPartyPriceBkColl.insert(rets)
|
||||
rets.clear()
|
||||
}
|
||||
}
|
||||
|
||||
def synchDataProcess(date){
|
||||
def q = new BasicDBObject()
|
||||
q.put("s_date", date)
|
||||
def dbcursor = dataProcessColl.find(q).batchSize(5000)
|
||||
def index = 0
|
||||
def rets = []
|
||||
def num = 0
|
||||
for(def d : dbcursor){
|
||||
num++
|
||||
index++
|
||||
rets << d
|
||||
if(index%10000==0){
|
||||
dataProcessBkColl.insert(rets)
|
||||
println(num)
|
||||
rets.clear()
|
||||
}
|
||||
}
|
||||
if(rets.size()>0){
|
||||
dataProcessBkColl.insert(rets)
|
||||
rets.clear()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
def addAnjukeRoad(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区地址拆分\\小区地址切分(结果)_1.xlsx", { d ->
|
||||
def roads = findThirdPartyPrice(d['A'], d['B'], d['C'], "链家")
|
||||
@@ -130,8 +153,6 @@ class AddressCut extends BaseBatch{
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
>>>>>>> c1a5ab8c6aa973a91e6be5547ec0925db3fa500f
|
||||
}
|
||||
|
||||
def sameAddr() {
|
||||
|
||||
+40
-40
@@ -1457,45 +1457,45 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
println "community_p_describe_gj 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportGuideStatistics(time){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []
|
||||
def allRows = []
|
||||
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def sql = null;
|
||||
if(CcbCommonUtils.citys_92.contains(city)){
|
||||
sql = "select MODEL_TYPE,CITYID,CITYNAME,YEAR_MONTH,COMM_SUM_COUNT,COMM_COUNT,RATIO,TYPE from GUIDE_STATISTICS where cityid='${cityCode}' and year_month=${time}";
|
||||
}else{
|
||||
sql = "select MODEL_TYPE,CITYID,CITYNAME,YEAR_MONTH,COMM_SUM_COUNT,COMM_COUNT,RATIO,TYPE from GUIDE_STATISTICS where cityid='${cityCode}'";
|
||||
}
|
||||
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
data << r.MODEL_TYPE
|
||||
data << r.CITYID
|
||||
data << r.CITYNAME
|
||||
data << r.YEAR_MONTH
|
||||
data << r.COMM_SUM_COUNT
|
||||
data << r.COMM_COUNT
|
||||
data << r.RATIO
|
||||
data << r.TYPE
|
||||
allRows << data
|
||||
});
|
||||
}
|
||||
println allRows.size()
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
|
||||
println "guide_statistics 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload("rh", fileName, curDate)
|
||||
println "guide_statistics 上传Ftp完成......"
|
||||
}
|
||||
// def exportGuideStatistics(time){
|
||||
//
|
||||
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
// def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
// if(CommonDatautils.isWindow()){
|
||||
// fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
// }
|
||||
// def headers = []
|
||||
// def allRows = []
|
||||
//
|
||||
// for(def city : citys) {
|
||||
// def cityCode = cityCodeMap.get(city)
|
||||
// def sql = null;
|
||||
// if(CcbCommonUtils.citys_92.contains(city)){
|
||||
// sql = "select MODEL_TYPE,CITYID,CITYNAME,YEAR_MONTH,COMM_SUM_COUNT,COMM_COUNT,RATIO,TYPE from GUIDE_STATISTICS where cityid='${cityCode}' and year_month=${time}";
|
||||
// }else{
|
||||
// sql = "select MODEL_TYPE,CITYID,CITYNAME,YEAR_MONTH,COMM_SUM_COUNT,COMM_COUNT,RATIO,TYPE from GUIDE_STATISTICS where cityid='${cityCode}'";
|
||||
// }
|
||||
//
|
||||
// sqlInstance.eachRow(sql, { r ->
|
||||
// def data = []
|
||||
// data << r.MODEL_TYPE
|
||||
// data << r.CITYID
|
||||
// data << r.CITYNAME
|
||||
// data << r.YEAR_MONTH
|
||||
// data << r.COMM_SUM_COUNT
|
||||
// data << r.COMM_COUNT
|
||||
// data << r.RATIO
|
||||
// data << r.TYPE
|
||||
// allRows << data
|
||||
// });
|
||||
// }
|
||||
// println allRows.size()
|
||||
// FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
//
|
||||
// println "guide_statistics 数据文件输出完成......"
|
||||
// CcbFtpUtils.ftpUpload("rh", fileName, curDate)
|
||||
// println "guide_statistics 上传Ftp完成......"
|
||||
// }
|
||||
|
||||
|
||||
def exportPgGuideStatistics(time){
|
||||
@@ -1514,7 +1514,6 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
data << r.gujia_model_type
|
||||
data << r.urbn_id
|
||||
data << r.urbn_nm
|
||||
data << r.gujia_clc_yrmo
|
||||
@@ -1522,6 +1521,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
data << r.gujia_comm_count
|
||||
data << r.gujia_ratio
|
||||
data << r.gujia_type
|
||||
data << r.gujia_model_type
|
||||
allRows << data
|
||||
});
|
||||
}
|
||||
|
||||
+240
-46
@@ -1,43 +1,22 @@
|
||||
package com.fangjia.script.commData
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.datastore.driver.FacetResult
|
||||
import com.fangjia.httpclient.UserAgents
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.JacksonUtil
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.AggregationOutput
|
||||
import com.mongodb.client.MongoCursor
|
||||
import com.mongodb.client.model.Accumulators
|
||||
import com.mongodb.client.model.Aggregates
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import groovy.json.JsonSlurper
|
||||
|
||||
import net.sf.json.JSONArray
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.poi.ss.util.RegionUtil
|
||||
import org.bson.Document
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import com.fangjia.core.util.HttpRequest;
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.datastore.driver.DSCollection
|
||||
import com.fangjia.datastore.driver.DSCursor
|
||||
import com.fangjia.datastore.driver.DataStore
|
||||
import com.fangjia.httpclient.UserAgents
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.newDistrict.InvokeHttpRequest
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.GeoUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBCursor
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
import com.fangjia.script.util.*
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.*
|
||||
import groovy.json.JsonSlurper
|
||||
import net.sf.json.JSONArray
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
@@ -122,6 +101,8 @@ class CommonDatautils {
|
||||
public static final DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
public static final DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com")
|
||||
|
||||
public static final DBCollection STATS_INDEX_COLL = MongoUtils.getCollectionByName("fangjia_stat", "stats_index", "mongo.db.fangjia.com")
|
||||
|
||||
public static DSCollection getHouseDataStoreConnection(){
|
||||
if(house_dataStoreConnection != null) {
|
||||
return house_dataStoreConnection
|
||||
@@ -820,6 +801,7 @@ class CommonDatautils {
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("status",0)
|
||||
// q.put("status",new BasicDBObject('$ne', 0))
|
||||
q.put("cat", cat);
|
||||
q.put("city", city)
|
||||
|
||||
@@ -829,7 +811,7 @@ class CommonDatautils {
|
||||
keys.put(field, 1);
|
||||
}
|
||||
}
|
||||
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(2000)
|
||||
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(5000)
|
||||
def rets = []
|
||||
for(def d : dbsour){
|
||||
rets << d
|
||||
@@ -959,13 +941,7 @@ class CommonDatautils {
|
||||
keys.put(field, 1);
|
||||
}
|
||||
}
|
||||
def cursor = getDataStoreConnection().find(q, keys).batchSize(2000)
|
||||
if(cursor){
|
||||
for(def obj : cursor){
|
||||
return obj;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
return SEAWEED_COLL.findOne(q, keys)
|
||||
}
|
||||
|
||||
|
||||
@@ -1132,6 +1108,10 @@ class CommonDatautils {
|
||||
q.put("city", city);
|
||||
q.put("cat", cat);
|
||||
q.put("name", name);
|
||||
if(region){
|
||||
q.put("region", region);
|
||||
}
|
||||
q.put("status",new BasicDBObject('$ne', 0))
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -1680,7 +1660,7 @@ class CommonDatautils {
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
queryCondition.put("city", city);
|
||||
queryCondition.put("category","region");
|
||||
queryCondition.put("show",1);
|
||||
// queryCondition.put("show",1);
|
||||
def dbCursor = REGION_BLOCK_COLL.find(queryCondition)
|
||||
def regions = [];
|
||||
for(DBObject obj : dbCursor){
|
||||
@@ -2385,6 +2365,9 @@ class CommonDatautils {
|
||||
q.put("city", city);
|
||||
if(region){
|
||||
q.put("region", region);
|
||||
q.put("cat", "区域")
|
||||
}else{
|
||||
q.put("cat", "城市")
|
||||
}
|
||||
q.put('weekend', weekend);
|
||||
def dbCursor = REGION_STAT_COLL.find(q).sort(new BasicDBObject().append("weekend", -1)).limit(1);
|
||||
@@ -3542,6 +3525,20 @@ class CommonDatautils {
|
||||
return prices;
|
||||
}
|
||||
|
||||
static def queryCityPriceByWeekend(city, weekend){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
queryCondition.put("city", city);
|
||||
queryCondition.put("cat", "城市");
|
||||
|
||||
queryCondition.put('weekend', DateUtils.parseFromTightYYYYMMDD(weekend));
|
||||
def obj = REGION_STAT_COLL.findOne(queryCondition)
|
||||
def prices = null;
|
||||
if(obj) {
|
||||
prices = obj.get("avg_price");
|
||||
}
|
||||
return prices;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -3735,9 +3732,9 @@ class CommonDatautils {
|
||||
if(!region){
|
||||
if(city){
|
||||
q.put("cat", "city")
|
||||
}else{
|
||||
q.put("cat", "region")
|
||||
}
|
||||
}else{
|
||||
q.put("cat", "region")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4095,6 +4092,21 @@ class CommonDatautils {
|
||||
return retMap;
|
||||
}
|
||||
|
||||
|
||||
static def findHouseCount(def city, def region, def name){
|
||||
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
|
||||
.must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("district_name", name)).must(QueryBuilders.termQuery("cat", "sell"))
|
||||
|
||||
|
||||
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
|
||||
def index = new Random().nextInt(5);
|
||||
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
|
||||
return houses.size();
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 是否是别墅
|
||||
* @return
|
||||
@@ -4766,7 +4778,6 @@ class CommonDatautils {
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","区域")
|
||||
q.put("district_diff",1)
|
||||
q.put("city",new BasicDBObject('$in', ["深圳市"]))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("province",1)
|
||||
@@ -4795,6 +4806,90 @@ class CommonDatautils {
|
||||
return regionMap
|
||||
}
|
||||
|
||||
|
||||
static def queryAllRegionNameByCity(){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","区域")
|
||||
q.put("district_diff",1)
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city_short",1)
|
||||
keys.put("district_short",1)
|
||||
keys.put("alias",1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def retMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def regionMap = retMap.get(d.city_short)
|
||||
if(!regionMap){
|
||||
regionMap = [:]
|
||||
retMap.put(d.city_short, regionMap)
|
||||
}
|
||||
regionMap.put(d.district_short, d.alias)
|
||||
}
|
||||
|
||||
return retMap
|
||||
}
|
||||
|
||||
static def getRegionShortName(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","区域")
|
||||
q.put("district_diff",1)
|
||||
q.put("city", city)
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("district",1)
|
||||
keys.put("district_short", 1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def shortNameMap = [:]
|
||||
for(def d : dbcursor){
|
||||
shortNameMap.put(d.district_short, d.district)
|
||||
}
|
||||
return shortNameMap
|
||||
}
|
||||
|
||||
|
||||
static def queryAllRegionFullNameByCity(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","区域")
|
||||
q.put("district_diff",1)
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city",1)
|
||||
keys.put("district",1)
|
||||
keys.put("alias",1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def retMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def regionMap = retMap.get(d.city)
|
||||
if(!regionMap){
|
||||
regionMap = [:]
|
||||
retMap.put(d.city, regionMap)
|
||||
}
|
||||
regionMap.put(d.district, d.alias)
|
||||
}
|
||||
|
||||
return retMap
|
||||
}
|
||||
|
||||
static def queryAllStreetNameByCity(city){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","乡镇街道")
|
||||
q.put("city", city)
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("street",1)
|
||||
keys.put("alias",1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def retMap = [:]
|
||||
for(def d : dbcursor){
|
||||
retMap.put(d.street, d.alias)
|
||||
}
|
||||
|
||||
return retMap
|
||||
}
|
||||
|
||||
static def queryCommunityByCity(){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
@@ -4825,7 +4920,72 @@ class CommonDatautils {
|
||||
return townMap
|
||||
}
|
||||
|
||||
static def queryMzbRegion(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","区域")
|
||||
q.put("city",city)
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("district",1)
|
||||
keys.put("mzb_district_name",1)
|
||||
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def regionMap = [:]
|
||||
for(def d : dbcursor){
|
||||
regionMap.put(d.district, d.mzb_district_name)
|
||||
}
|
||||
return regionMap
|
||||
}
|
||||
|
||||
static def queryTownByCity(city, fields, mzbRegionMap){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","乡镇街道")
|
||||
q.put("city",city)
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
for(def key : fields){
|
||||
keys.put(key,1)
|
||||
}
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def townMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = mzbRegionMap.get(d.district)
|
||||
def list = townMap.get(key)
|
||||
if(!list){
|
||||
list = []
|
||||
}
|
||||
list << d
|
||||
townMap.put(key, list)
|
||||
}
|
||||
|
||||
println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss")
|
||||
return townMap
|
||||
}
|
||||
|
||||
static def queryTownByCity(city, fields){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","乡镇街道")
|
||||
q.put("city",city)
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
for(def key : fields){
|
||||
keys.put(key,1)
|
||||
}
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def townMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def list = townMap.get(d.district)
|
||||
if(!list){
|
||||
list = []
|
||||
}
|
||||
list << d
|
||||
townMap.put(d.district, list)
|
||||
}
|
||||
|
||||
println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss")
|
||||
return townMap
|
||||
}
|
||||
|
||||
static def queryTownByCity(city){
|
||||
|
||||
@@ -4861,7 +5021,7 @@ class CommonDatautils {
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","乡镇街道")
|
||||
q.put("city",new BasicDBObject('$in', ["深圳"]))
|
||||
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city",1)
|
||||
@@ -5060,4 +5220,38 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
static def findStatIndexBySort(type, city, region, sort){
|
||||
def q = new BasicDBObject()
|
||||
q.put("cat", "district")
|
||||
q.put("type", type)
|
||||
if(region){
|
||||
q.put("district", region)
|
||||
}
|
||||
if(city){
|
||||
q.put("city", city)
|
||||
}
|
||||
|
||||
def cur = STATS_INDEX_COLL.find(q).sort(new BasicDBObject("yearmonth", sort)).limit(1)
|
||||
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def obj = 0;
|
||||
while(cur.hasNext()) {
|
||||
obj = cur.next()
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
static def findStatIndexByMonth(type, city, region, month){
|
||||
def q = new BasicDBObject()
|
||||
q.put("cat", "district")
|
||||
q.put("type", type)
|
||||
if(region){
|
||||
q.put("district", region)
|
||||
}
|
||||
if(city){
|
||||
q.put("city", city)
|
||||
}
|
||||
q.put("yearmonth", month)
|
||||
return STATS_INDEX_COLL.findOne(q)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -2,8 +2,10 @@ package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.core.BlockData
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
@@ -531,6 +533,28 @@ class FiledStrand {
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
static def processLngAndLat(val) {
|
||||
if (val) {
|
||||
def map = [:]
|
||||
def lng = DataHandleUtils.getDoubleRound(6, val.x)
|
||||
def lat = DataHandleUtils.getDoubleRound(6, val.y)
|
||||
Long gd_lat = lat * 10000000000L
|
||||
Long gd_lng = lng * 10000000000L
|
||||
map.lat = gd_lat
|
||||
map.lng = gd_lng
|
||||
|
||||
def baidu = BaiDuApiUtils.geoconv(lng, lat, 3)
|
||||
Long bd_lng = baidu.x * 10000000000L
|
||||
Long bd_lat = baidu.y * 10000000000L
|
||||
map.lat2 = bd_lat
|
||||
map.lng2 = bd_lng
|
||||
return map
|
||||
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def isRoad(def s){
|
||||
def pattern = Pattern.compile("[·|½Ö|´óµÀ|Ïï]");
|
||||
def matcher = pattern.matcher(s)
|
||||
|
||||
@@ -2,33 +2,97 @@ package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\成交未匹配.xlsx",{d->
|
||||
if(d['A'] && !d['E']){
|
||||
// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'].toString(), "district")
|
||||
// def dist = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'].toString(), "district")
|
||||
println d['A'] +" "+d['C']
|
||||
// def total = findDatas(d['A'], d['B'], d['C'])
|
||||
// d['D'] = total
|
||||
def c = getAmapLocation(d['A']+"市", d['B']+"区", d['C'].replace("?","").replace("·","").replace("?","").replace("|","").replace("·",""))
|
||||
d['E'] = c
|
||||
}
|
||||
})
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
//ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\成交未匹配.xlsx",{d->
|
||||
// if(d['A'] && !d['E']){
|
||||
//// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'].toString(), "district")
|
||||
//// def dist = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'].toString(), "district")
|
||||
// println d['A'] +" "+d['C']
|
||||
//// def total = findDatas(d['A'], d['B'], d['C'])
|
||||
//// d['D'] = total
|
||||
// def c = getAmapLocation(d['A']+"市", d['B']+"区", d['C'].replace("?","").replace("·","").replace("?","").replace("|","").replace("·",""))
|
||||
// d['E'] = c
|
||||
// }
|
||||
//})
|
||||
|
||||
def URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection seaweed_coll = MongoUtils.getCollectionByName2("fangjia", "seaweed", URL)
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", "上海市")
|
||||
q.put("category", "district")
|
||||
q.put("status", 0)
|
||||
def ret = []
|
||||
def list = seaweed_coll.find(q).batchSize(5000).toList()
|
||||
println "总计:"+list.size()
|
||||
//for(def l : list){
|
||||
// def obj = getAmapLocation(l.city, l.region, l.name)
|
||||
// if(obj){
|
||||
// ret << [l.city, l.region, l.name, obj.cityname, obj.adname, obj.name,obj.type]
|
||||
// }
|
||||
//}
|
||||
|
||||
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
def addCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(list.size())
|
||||
def total = list.size()
|
||||
for(def i=0;i<list.size();i++){
|
||||
final def l = list[i]
|
||||
def dist = CommonDatautils.findDistrict(l.fangjia_city, l.fangjia_region, l.name, "district")
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
addCount.addAndGet(1)
|
||||
|
||||
def alias = []
|
||||
alias << dist.name
|
||||
if(l.alias){
|
||||
alias.addAll(l.alias)
|
||||
}
|
||||
for(def alia : alias){
|
||||
def obj = getAmapLocation(l.city, l.region, alia)
|
||||
if(obj){
|
||||
ret << [l.city, l.region, l.name, obj.cityname, obj.adname, obj.name,obj.type]
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
println addCount.get()+"/"+total+" "+ ret.size()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
|
||||
ExcelUtils.write("D:\\数据\\数据建设\\小区\\高德地图有围栏数据.xlsx", ret, "高德")
|
||||
|
||||
|
||||
|
||||
|
||||
def getAmapLocation(city, region, name){
|
||||
String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') "
|
||||
def total = 0
|
||||
String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
|
||||
def obj = null
|
||||
ElasticUtils.executeSqlQuery(sql,{ d->
|
||||
if(d.name.contains(name) && d.type.contains("住宅")){
|
||||
total ++
|
||||
if(d.name.contains(name) && d.type.contains("住宅") && d.shape){
|
||||
obj = d
|
||||
}
|
||||
})
|
||||
return total;
|
||||
return obj;
|
||||
}
|
||||
|
||||
|
||||
|
||||
+112
-3
@@ -6,21 +6,33 @@ import com.fangjia.script.averager.InsertMonitorSquareRentCalcCommandBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.MongoClient
|
||||
import com.rabbitmq.client.AMQP
|
||||
|
||||
class AddCityRegionPrice {
|
||||
|
||||
public final static DBCollection COMMAND_COLL = new MongoClient("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('calc_command')
|
||||
|
||||
|
||||
public static final def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName2("fangjia", "sqmetre_rent", mongoDbUrl)
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", mongoDbUrl)
|
||||
|
||||
static main(args) {
|
||||
def task = new AddCityRegionPrice()
|
||||
// task.addRegionPrice()
|
||||
// task.addCityprice()
|
||||
task.addCityprice()
|
||||
// task.addDistrictprice()
|
||||
// task.addCityRentPrice()
|
||||
task.addDistrictRentprice()
|
||||
// task.addRegionRentPrice()
|
||||
// task.addDistrictRentprice()
|
||||
|
||||
// task.updateRegionRentData()
|
||||
// task.updateDealPriceAreaType()
|
||||
// task.checkCityImport()
|
||||
}
|
||||
|
||||
def addRegionPrice(){
|
||||
@@ -54,7 +66,7 @@ class AddCityRegionPrice {
|
||||
|
||||
def addCityprice(){
|
||||
InsertMonitorCalcCommandBatch insertBatch = new InsertMonitorCalcCommandBatch()
|
||||
def citys = CommonDatautils.findAllCity().reverse()
|
||||
def citys = ["乐东"]// CommonDatautils.findAllCity().reverse()
|
||||
def add = 0
|
||||
for(def c : citys){
|
||||
def startObj = CommonDatautils.queryCityOrRegionPrice(c, null, 1)
|
||||
@@ -125,6 +137,35 @@ class AddCityRegionPrice {
|
||||
}
|
||||
}
|
||||
|
||||
def addRegionRentPrice(){
|
||||
InsertMonitorSquareRentCalcCommandBatch inserBatch = new InsertMonitorSquareRentCalcCommandBatch();
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
for(def c : citys){
|
||||
|
||||
def regions = CommonDatautils.getCityRegions(c)
|
||||
for(def r : regions){
|
||||
// if(r!="南川"){
|
||||
// continue
|
||||
// }
|
||||
def startObj = CommonDatautils.queryFirstOrLastRentPrice(c, r, null, 1)
|
||||
def endObj = CommonDatautils.queryFirstOrLastRentPrice(c, r, null, -1)
|
||||
if(startObj && endObj){
|
||||
def months = DateUtils.getBetweenMonth(DateUtil.formatByPattern(startObj.weekend, "yyyyMM"),"202409")
|
||||
months.each {m->
|
||||
// println m
|
||||
def lastDay = DateUtils.truncateDay(DateUtils.getLastDayOfMonth(DateUtil.parseDate(m+"01", "yyyyMMdd")))
|
||||
def obj = CommonDatautils.findSqmetreRent(c, r, null, lastDay)
|
||||
if(!obj){
|
||||
inserBatch.saveCityCalcCommand(c, r, null, lastDay, "fangjia-priceanalysis-0")
|
||||
println "[add] "+c+" "+r+" "+DateUtil.formatByPattern(lastDay, "yyyy-MM-dd")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def addDistrictRentprice(){
|
||||
InsertMonitorSquareRentCalcCommandBatch inserBatch = new InsertMonitorSquareRentCalcCommandBatch();
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
@@ -158,4 +199,72 @@ class AddCityRegionPrice {
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def updateRegionRentData(){
|
||||
def q = new BasicDBObject("cat", "region")
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("markup", 1)
|
||||
def dbcursor = SQMETRE_RENT_COLL.find(q, keys).batchSize(5000)
|
||||
def index = 1
|
||||
for(def d : dbcursor){
|
||||
if(!(d.markup instanceof Double)){
|
||||
def udata = new BasicDBObject()
|
||||
double markup = 1.0
|
||||
markup = d.markup * markup
|
||||
udata.put("markup", markup)
|
||||
SQMETRE_RENT_COLL.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
println index
|
||||
index++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def updateDealPriceAreaType(){
|
||||
def index = 1
|
||||
def citys = ["凉山州","福州","南京","佛山","沈阳","苏州"]
|
||||
citys.each {c->
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", c)
|
||||
// q.put("price_flag", "国信达成交-推导")
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
for(def d : dbcursor){
|
||||
// if(!(d.area instanceof Double)){
|
||||
// def udata = new BasicDBObject()
|
||||
// double area = 1.0
|
||||
// area = d.area * area
|
||||
// udata.put("area", area)
|
||||
// DEAL_PRICE_COLL.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
// println c+" "+index
|
||||
// index++
|
||||
// }
|
||||
if(!(d.total_price instanceof Integer)){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("total_price", d.total_price.toString().toDouble().toInteger())
|
||||
DEAL_PRICE_COLL.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
println c+" "+index
|
||||
index++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def checkCityImport(){
|
||||
def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]
|
||||
for(def c : citys){
|
||||
def startObj = CommonDatautils.findStatIndexBySort("城市", c, null, 1)
|
||||
def endObj = CommonDatautils.findStatIndexBySort("城市", c,null, -1)
|
||||
if(startObj && endObj){
|
||||
def months = DateUtils.getBetweenMonth(startObj.yearmonth.toString(),"202409")
|
||||
months.each {m->
|
||||
def obj = CommonDatautils.findStatIndexByMonth("城市", c,null, m.toInteger())
|
||||
if(!obj){
|
||||
println "[add] "+c+" "+m
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println c +" 一个也没有"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+376
@@ -0,0 +1,376 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import groovy.json.JsonSlurper
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 添加高德地图,百度地图围栏
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class AddDistrictShapeBatch extends BaseBatch{
|
||||
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
|
||||
|
||||
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
|
||||
public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com")
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new AddDistrictShapeBatch(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
// findGaoDeDatas2("苏州市","吴中区", "水岸清华瀚宫" )
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def index = 1
|
||||
for(def c : citys){
|
||||
println c +" "+index
|
||||
index ++
|
||||
def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2"])
|
||||
// baiduMatch(c, districts)
|
||||
gaodeMatch(c, districts)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def baiduMatch(city,districts){
|
||||
def data = fullCityMap.get(city)
|
||||
def fullCityName = null
|
||||
if(data){
|
||||
fullCityName = data.city
|
||||
}
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
if(fullCityName){
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def d = districts[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def baiduId = baiduProcess(d, fullCityName)
|
||||
if(baiduId){
|
||||
if(!rets.contains(baiduId)){
|
||||
rets << baiduId
|
||||
}else{
|
||||
println "百度围栏 "+baiduId +" 被:["+d.city+" "+d.region+" "+d.name+" ]再次匹配,查看原始小区是否重复或别名相同"
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}else{
|
||||
println city +" 找不到相应的全称"
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
|
||||
def baiduProcess(d, fullCityName){
|
||||
def alias = []
|
||||
alias << d.name
|
||||
if(d.alias){
|
||||
alias.addAll(d.alias)
|
||||
}
|
||||
|
||||
def distData = mcaDistMap.get(d.city+"#"+d.region)
|
||||
def fullDistName = null
|
||||
if(distData){
|
||||
fullDistName = distData.district
|
||||
}
|
||||
|
||||
def fenceMap = null
|
||||
for(def name : alias){
|
||||
fenceMap = findBaiduDatas(fullCityName, fullDistName, name)
|
||||
if(!fenceMap){
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if(fenceMap){
|
||||
def o = new BasicDBObject()
|
||||
o.put("city", d.city)
|
||||
o.put("district", d.region)
|
||||
o.put("name", d.name)
|
||||
o.put("source", "百度地图")
|
||||
o.put("c_date", new Date())
|
||||
o.put("data_id", fenceMap.data_id)
|
||||
o.put("comm_id", d._id.toString())
|
||||
def resultMap = [:]
|
||||
resultMap.coordinates = fenceMap.coordinatePairs
|
||||
// if(fenceMap.coordinatePairs.size()>1) {
|
||||
// resultMap.type = "MultiPolygon"
|
||||
// }else{
|
||||
resultMap.type = "polygon"
|
||||
// }
|
||||
o.put("shape", resultMap)
|
||||
|
||||
def q = new BasicDBObject("comm_id", d._id.toString())
|
||||
seaweed_fence_coll.update(q, new BasicDBObject('$set', o),true,false)
|
||||
// seaweed_fence_coll.save(o)
|
||||
println "[save] 围栏 ( "+d.city +" "+d.region +" "+d.name+" )"
|
||||
return fenceMap.data_id
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def findBaiduDatas(city, district, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("district", district)
|
||||
q.put("name", name)
|
||||
q.put('std_tag', '房地产;住宅区')
|
||||
q.put("guoke_geo", new BasicDBObject('$ne', null))
|
||||
def dbcursor = BAIDU_POI_COLL.find(q)
|
||||
def map = [:]
|
||||
println dbcursor.count()
|
||||
for(def d : dbcursor){
|
||||
def arrs = d.gcj02_guoke_geo.split("\\|")
|
||||
|
||||
def coordinatesStr = arrs[2].toString().substring(2, arrs[2].toString().size()-1 )
|
||||
// 将字符串分割成列表
|
||||
def coordinatePairs = coordinatesStr.split(',')
|
||||
def coordinatesList = []
|
||||
for (int i = 0; i < coordinatePairs.size(); i += 2) {
|
||||
// 将字符串坐标转换为浮点数
|
||||
double longitude = coordinatePairs[i] as double
|
||||
double latitude = coordinatePairs[i + 1] as double
|
||||
|
||||
// 将坐标点作为一个元组添加到集合中
|
||||
coordinatesList << [longitude, latitude]
|
||||
}
|
||||
if(coordinatePairs){
|
||||
map.data_id = d._id
|
||||
map.coordinatePairs = coordinatesList
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
def gaodeMatch(city,districts){
|
||||
def data = fullCityMap.get(city)
|
||||
def fullCityName = null
|
||||
if(data){
|
||||
fullCityName = data.city
|
||||
}
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
if(fullCityName){
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def d = districts[i]
|
||||
def fence = findSeaweedFenceExists(d._id.toString())
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
if(!fence){
|
||||
def baiduId = gaodeProcess(d, fullCityName)
|
||||
if(baiduId){
|
||||
if(!rets.contains(baiduId)){
|
||||
rets << baiduId
|
||||
}else{
|
||||
println "高德围栏 "+baiduId +" 被:["+d.city+" "+d.region+" "+d.name+" ]再次匹配,查看原始小区是否重复或别名相同"
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}else{
|
||||
println city +" 找不到相应的全称"
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
|
||||
def gaodeProcess(d, fullCityName){
|
||||
def alias = []
|
||||
alias << d.name
|
||||
if(d.alias){
|
||||
alias.addAll(d.alias)
|
||||
}
|
||||
|
||||
def distData = mcaDistMap.get(d.city+"#"+d.region)
|
||||
def fullDistName = null
|
||||
if(distData){
|
||||
fullDistName = distData.district
|
||||
}
|
||||
|
||||
def fenceMap = null
|
||||
for(def name : alias){
|
||||
fenceMap = findGaoDeDatas2(fullCityName, fullDistName, name)
|
||||
if(!fenceMap){
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if(fenceMap){
|
||||
def o = new BasicDBObject()
|
||||
o.put("city", d.city)
|
||||
o.put("district", d.region)
|
||||
o.put("name", d.name)
|
||||
o.put("source", "高德地图")
|
||||
o.put("c_date", new Date())
|
||||
o.put("data_id", fenceMap.id)
|
||||
o.put("comm_id", d._id.toString())
|
||||
def resultMap = [:]
|
||||
// def shape = fenceMap.shape
|
||||
// def points_x = fenceMap.shape.polygon.x
|
||||
// def points_y = fenceMap.shape.polygon.y
|
||||
// def list = []
|
||||
// for(def i=0;i<points_x.size();i++){
|
||||
// list << [points_x[i], points_y[i]]
|
||||
// }
|
||||
resultMap.coordinates = fenceMap.coordinates
|
||||
resultMap.type = "polygon"
|
||||
o.put("shape", resultMap)
|
||||
|
||||
def q = new BasicDBObject("comm_id", d._id.toString())
|
||||
seaweed_fence_coll.update(q, new BasicDBObject('$set', o),true,false)
|
||||
// seaweed_fence_coll.save(o)
|
||||
println "[save] 高德围栏 ( "+d.city +" "+d.region +" "+d.name+" )"
|
||||
return fenceMap.data_id
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def findGaoDeDatas(city, region, name){
|
||||
String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
|
||||
def obj = null
|
||||
ElasticUtils.executeSqlQuery(sql,{ d->
|
||||
def newName = d.name.replace("-","").replace("·","")
|
||||
if(newName == name && d.type.contains("住宅") && d.shape){
|
||||
obj = d
|
||||
}
|
||||
})
|
||||
return obj;
|
||||
}
|
||||
|
||||
|
||||
def findGaoDeDatas2(city, region, name){
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:goojia123456".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.206:9200/amap_poi_detail/_search"
|
||||
def body = """
|
||||
{
|
||||
"_source": {
|
||||
"includes": ["name", "shape", "id"]
|
||||
},
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"term": {
|
||||
"cityname" : "${city}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"term": {
|
||||
"adname" : "${region}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
"name": {"query": "${name}","operator": "and"}
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
"type": "住宅"
|
||||
}
|
||||
},
|
||||
{
|
||||
"exists": {
|
||||
"field": "shape"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
def response = new JsonSlurper().parseText(res)
|
||||
def obj = null
|
||||
def fenceMap = [:]
|
||||
if(response && response.hits){
|
||||
def hits = response.hits.hits
|
||||
if(hits){
|
||||
def d = hits[0]
|
||||
def source = d._source
|
||||
def newName = source.name.replace("-","").replace("·","").replace("(建设中)","")
|
||||
if(newName == name){
|
||||
fenceMap.id = source.id
|
||||
def list = []
|
||||
def points_arr = source.shape.coordinates
|
||||
for(def points : points_arr){
|
||||
def rets = []
|
||||
for(def point : points){
|
||||
def lng = DataHandleUtils.getDoubleRound(6, point[0].toString().toDouble())
|
||||
def lat = DataHandleUtils.getDoubleRound(6, point[1].toString().toDouble())
|
||||
rets << [lng, lat]
|
||||
}
|
||||
list << rets
|
||||
}
|
||||
if(list.size()==1){
|
||||
fenceMap.coordinates = list[0]
|
||||
}else{
|
||||
fenceMap.coordinates = list
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
return fenceMap
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def findSeaweedFenceExists(commid){
|
||||
def q = new BasicDBObject()
|
||||
q.put("comm_id", commid)
|
||||
return seaweed_fence_coll.findOne(q)
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+234
@@ -0,0 +1,234 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 小区自动入库(新增小区,每月运行一次)
|
||||
*/
|
||||
class AutoAddDistrictBatch extends BaseBatch{
|
||||
|
||||
final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018)
|
||||
public static DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
|
||||
|
||||
static main(args) {
|
||||
runBatch(new AutoAddDistrictBatch(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
//查找第三方匹配没有匹配上的小区
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def list = []
|
||||
list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "高德地图"]
|
||||
for(def c : citys){
|
||||
def noMatchDatas = findNoMatchResult(c)
|
||||
println c +" 新旧算法未匹配的数据量:"+noMatchDatas.size()
|
||||
def rets = checkDatas(noMatchDatas.keySet())
|
||||
list.addAll(rets)
|
||||
}
|
||||
|
||||
def fileName = "/home/goojia/gj/add_dist_info.xlsx"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\数据建设\\小区\\第三方待入库数据.xlsx"
|
||||
}
|
||||
ExcelUtils.write(fileName, list, "待入库数据")
|
||||
}
|
||||
|
||||
def checkDatas(keysets){
|
||||
def noMatchCount = keysets.size()
|
||||
final CountDownLatch latch = new CountDownLatch(noMatchCount)
|
||||
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
for(def i=0;i<keysets.size();i++){
|
||||
final def key = keysets[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def arrs = key.split("##")
|
||||
//检测有没有成交
|
||||
def trade_count = findTrades(arrs[0], arrs[1], arrs[2])
|
||||
//检测有没有房源
|
||||
def house_count = CommonDatautils.findHouseCount(arrs[0], arrs[1], arrs[2])
|
||||
//检查百度地图有没有
|
||||
def baidu_count = findBaiduCount(arrs[0], arrs[1], arrs[2])
|
||||
//检查高德地图有没有
|
||||
def gaode_count = findGaodeCount(arrs[0], arrs[1], arrs[2])
|
||||
println "[ "+totalCount.get()+"/"+noMatchCount+" ]"+key +" trade_count:"+trade_count+" house_count:"+house_count+" baidu_count:"+baidu_count+" gaode_count:"+gaode_count
|
||||
rets << [arrs[0], arrs[1], arrs[2], trade_count, house_count, baidu_count, gaode_count]
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
def findBaiduCount(city, region, name){
|
||||
def data = fullCityMap.get(city)
|
||||
def fullCityName = null
|
||||
if(data){
|
||||
fullCityName = data.city
|
||||
}
|
||||
if(fullCityName){
|
||||
def distData = mcaDistMap.get(city+"#"+region)
|
||||
def fullDistName = null
|
||||
if(distData){
|
||||
fullDistName = distData.district
|
||||
}
|
||||
if(fullDistName){
|
||||
return findBaiduDatas(fullCityName, fullDistName, name)
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def findGaodeCount(city, region, name){
|
||||
def data = fullCityMap.get(city)
|
||||
def fullCityName = null
|
||||
if(data){
|
||||
fullCityName = data.city
|
||||
}
|
||||
if(fullCityName){
|
||||
def distData = mcaDistMap.get(city+"#"+region)
|
||||
def fullDistName = null
|
||||
if(distData){
|
||||
fullDistName = distData.district
|
||||
}
|
||||
if(fullDistName){
|
||||
def obj = findGaoDeDatas(fullCityName, fullDistName, name)
|
||||
if(obj){
|
||||
return 1
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
def findGaoDeDatas(city, region, name){
|
||||
String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
|
||||
def obj = null
|
||||
ElasticUtils.executeSqlQuery(sql,{ d->
|
||||
if(d.name == name && d.type.contains("住宅") && d.shape){
|
||||
obj = d
|
||||
}
|
||||
})
|
||||
return obj;
|
||||
}
|
||||
|
||||
def findTrades(city, region, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("district_name", name)
|
||||
return DEAL_PRICE_COLL.count(q)
|
||||
}
|
||||
|
||||
def findNoMatchResult(def c){
|
||||
def q = new BasicDBObject()
|
||||
q.put("match_id", new BasicDBObject('$exists',false))
|
||||
q.put("old_match_id", new BasicDBObject('$exists',false))
|
||||
q.put("city", c)
|
||||
def dbcursor = thirdSeaweedMappingColl.find(q).batchSize(5000)
|
||||
def rMap = [:]
|
||||
for(def d : dbcursor){
|
||||
if(d.estate_type){
|
||||
if(d.estate_type.contains("自建房")){
|
||||
continue
|
||||
}
|
||||
if(d.estate_type.contains("购物中心")){
|
||||
continue
|
||||
}
|
||||
if(d.estate_type.contains("住宅底商") || d.estate_type.contains("酒店") || d.estate_type.contains("商铺")){
|
||||
continue
|
||||
}
|
||||
if(d.estate_type.contains("办公") || d.estate_type.contains("写字楼")){
|
||||
continue
|
||||
}
|
||||
}else{
|
||||
continue
|
||||
}
|
||||
def key = d.city + "##"+d.district+"##"+d.name
|
||||
def rets = rMap.get(key)
|
||||
if(!rMap.containsKey(key)){
|
||||
rets = []
|
||||
rMap.put(key, rets)
|
||||
}
|
||||
rets << d
|
||||
}
|
||||
return rMap
|
||||
}
|
||||
|
||||
def createData(city, region, name, address, lng, lat){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("city", city)
|
||||
udata.put("region", region)
|
||||
name = name.replaceAll('[·\\(\\)()]|·', '').replace("建设中", "")
|
||||
udata.put("name", name)
|
||||
|
||||
def addr = AddressUtil.formatAddress(city, address)
|
||||
if(addr){
|
||||
udata.put("address", addr)
|
||||
}
|
||||
|
||||
udata.put("status", "0")
|
||||
udata.put("name_cat", "0")
|
||||
udata.put("cat", "district")
|
||||
udata.put("username", "高娟")
|
||||
|
||||
def estateType2 = "普通住宅"
|
||||
def type = ""
|
||||
if(type){
|
||||
if(type.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(type.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}
|
||||
}
|
||||
udata.put("estateType2", estateType2)
|
||||
if(lng){
|
||||
Long gd_lng = lng*10000000000
|
||||
udata.put("lng", gd_lng)
|
||||
}
|
||||
if(lat){
|
||||
Long gd_lat = lat * 10000000000
|
||||
udata.put("lat", gd_lat)
|
||||
}
|
||||
FangjiaClientUtils.saveOrUpdateDistrict(udata)
|
||||
}
|
||||
|
||||
def findBaiduDatas(city, district, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("district", district)
|
||||
q.put("name", name)
|
||||
q.put('std_tag', '房地产;住宅区')
|
||||
def count = BAIDU_POI_COLL.count(q)
|
||||
return count;
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -46,7 +46,7 @@ class DevProcess {
|
||||
def task = new DevProcess();
|
||||
// task.run()
|
||||
// task.parseCityAndDist()
|
||||
task.devSplitRoad()
|
||||
// task.devSplitRoad()
|
||||
// task.devNameSplitRoad()
|
||||
// task.addFlag()
|
||||
// task.matchDev()
|
||||
@@ -63,7 +63,7 @@ class DevProcess {
|
||||
// task.createDevMapping()
|
||||
// task.deleteAddr()
|
||||
// task.deleteAddr2()
|
||||
// task.synchDevTo100()
|
||||
task.synchDevTo100()
|
||||
// task.addNewDev2()
|
||||
|
||||
// task.addAlias()
|
||||
|
||||
+10
-2
@@ -20,6 +20,10 @@ class DevUpdateTestBatch {
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection development_zone_43_CONN = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
DBCollection development_zone_100_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
|
||||
|
||||
DBCollection DEV_MAPPING_100_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
|
||||
|
||||
|
||||
@@ -42,7 +46,7 @@ class DevUpdateTestBatch {
|
||||
def addDev(path){
|
||||
ExcelUtils.modify(path,{ d->
|
||||
if(!d['I'] && d['A']){
|
||||
def obj = development_zone_100_CONN.findOne(new BasicDBObject("name", d['A']))
|
||||
def obj = development_zone_43_CONN.findOne(new BasicDBObject("name", d['A']))
|
||||
if(!obj){
|
||||
def o = new BasicDBObject()
|
||||
o.put("province", d['B'])
|
||||
@@ -118,6 +122,8 @@ class DevUpdateTestBatch {
|
||||
q.put("address", d['C'])
|
||||
}else if(d['B'] == "Ô°Çø"){
|
||||
q.put("park", d['C'])
|
||||
}else if(d['B'] == "Poi"){
|
||||
q.put("poi", d['C'])
|
||||
}
|
||||
def mappingObj = DEV_MAPPING_100_CONN.findOne(q)
|
||||
if(!mappingObj){
|
||||
@@ -133,6 +139,8 @@ class DevUpdateTestBatch {
|
||||
o.put("address", d['C'])
|
||||
}else if(d['B'] == "Ô°Çø"){
|
||||
o.put("park", d['C'])
|
||||
}else if(d['B'] == "Poi"){
|
||||
o.put("poi", d['C'])
|
||||
}
|
||||
def date = new Date()
|
||||
o.put("c_date", date)
|
||||
@@ -160,7 +168,7 @@ class DevUpdateTestBatch {
|
||||
println d['A'] +" ²»´æÔÚ"
|
||||
}
|
||||
}
|
||||
},1)
|
||||
},2)
|
||||
}
|
||||
|
||||
|
||||
|
||||
+27
-1
@@ -28,7 +28,7 @@ class GXDdistrictMatch {
|
||||
static main(args) {
|
||||
def task = new GXDdistrictMatch();
|
||||
// task.run()
|
||||
task.addOldMatch()
|
||||
task.addOldMatch2()
|
||||
// task.runDb()
|
||||
|
||||
// task.updateDistStatus()//恢复name_cat不正常数据
|
||||
@@ -68,6 +68,32 @@ class GXDdistrictMatch {
|
||||
|
||||
}
|
||||
|
||||
def addOldMatch2(){
|
||||
//老匹配
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\建行业务反馈badcase.xlsx"//11城重新匹配.xlsx
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
println d['C']
|
||||
if(!d['AJ']){
|
||||
def jsonObject = OpenApiClient.addressMatch(d['C'], d['D'], "district", false,null, d['D'])
|
||||
if (jsonObject != null && !jsonObject.containsKey("result")) {
|
||||
d['AJ'] = jsonObject.get("id")
|
||||
d['AK'] = jsonObject.get("city")
|
||||
d['AL'] = jsonObject.get("district")
|
||||
d['AM'] = jsonObject.get("name")
|
||||
d['AN'] = jsonObject.get("address")
|
||||
|
||||
def dbo = CommonDatautils.findDistrictById(jsonObject.get("id"))
|
||||
if(dbo){
|
||||
d['AO'] = dbo.status
|
||||
d['AP'] = dbo.name_cat
|
||||
d['AQ'] = dbo.estate_type2
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
def runDb(){
|
||||
def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx"
|
||||
ExcelUtils.modify(fileName, { d ->
|
||||
|
||||
+350
@@ -0,0 +1,350 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.update.UpdateRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
/**
|
||||
* 区域标准化
|
||||
* 1.小区库区域标准化,包含小区,商铺,写字楼
|
||||
* 2.poi数据库中 区域标准化
|
||||
*/
|
||||
class RegionStandardBatch extends BaseBatch{
|
||||
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
|
||||
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
|
||||
|
||||
static main(args) {
|
||||
runBatch(new RegionStandardBatch(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
// seaweedRegionStandard()
|
||||
poiRegionStandard()
|
||||
}
|
||||
|
||||
def seaweedRegionStandard(){
|
||||
seaweedRegionStandardByCat("district")
|
||||
}
|
||||
def poiRegionStandard(){
|
||||
def runCitys = ["重庆","三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德"]
|
||||
def index = 0
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
for(def city : citys){
|
||||
index ++
|
||||
if(index<32){
|
||||
continue
|
||||
}
|
||||
if(runCitys.contains(city)){
|
||||
continue
|
||||
}
|
||||
def data = fullCityMap.get(city)
|
||||
def fullCityName = null
|
||||
if(data){
|
||||
fullCityName = data.city
|
||||
}
|
||||
if(fullCityName){
|
||||
|
||||
def rMap = regionFullNameMap.get(fullCityName)
|
||||
if(rMap){
|
||||
def set = rMap.keySet()
|
||||
if(set.size()>1){
|
||||
def rets = findEsPoiByCity(fullCityName)
|
||||
println city +" "+rets.size()
|
||||
poiHaveRegionProcess(city, set, rMap, rets)
|
||||
}else{
|
||||
// haveStreetProcess(city, set, cat)
|
||||
}
|
||||
}else{
|
||||
println "没找区域:"+city
|
||||
}
|
||||
}else{
|
||||
println "没找到相应的全称:"+city
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
def seaweedRegionStandardByCat(cat){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
for(def city : citys){
|
||||
|
||||
def rMap = regionMap.get(city)
|
||||
if(rMap){
|
||||
def set = rMap.keySet()
|
||||
if(set.size()>1){
|
||||
haveRegionProcess(city, set, rMap, cat)
|
||||
}else{
|
||||
haveStreetProcess(city, set, cat)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def haveRegionProcess(city, set, rMap, cat){
|
||||
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name"])
|
||||
for(def d : districts){
|
||||
if(set.contains(d.region)){
|
||||
continue
|
||||
}
|
||||
def updateRegion = null
|
||||
for(def key : set){
|
||||
def alias = rMap.get(key)
|
||||
if(alias.contains(d.region)){
|
||||
updateRegion = key
|
||||
break;
|
||||
}
|
||||
}
|
||||
if(updateRegion){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("region", updateRegion)
|
||||
udata.put("m_date", new Date())
|
||||
println d.city +" "+d.region+" "+d.name +" 区域修改为:"+updateRegion
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("status", 1)
|
||||
udata.put("m_date", new Date())
|
||||
println d.city +" "+d.region+" "+d.name +" 状态修改为:删除"
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def haveStreetProcess(city, set, cat){
|
||||
def data = fullCityMap.get(city)
|
||||
def fullCityName = null
|
||||
if(data){
|
||||
fullCityName = data.city
|
||||
}
|
||||
def streetMap = CommonDatautils.queryAllStreetNameByCity(fullCityName)
|
||||
if(streetMap){
|
||||
set = streetMap.keySet()
|
||||
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name"])
|
||||
for(def d : districts){
|
||||
if(set.contains(d.region)){
|
||||
continue
|
||||
}
|
||||
|
||||
def updateRegion = null
|
||||
def flag = null
|
||||
for(def key : set){
|
||||
def alias = streetMap.get(key)
|
||||
if((alias && alias.contains(d.region)) || (!alias && key.contains(d.region))){
|
||||
def newRegion = RegionData.formatRegion(d.city, key)
|
||||
if(newRegion!=d.region){
|
||||
updateRegion = newRegion
|
||||
}
|
||||
flag = "pass"
|
||||
break;
|
||||
}
|
||||
}
|
||||
if(updateRegion){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("region", updateRegion)
|
||||
udata.put("m_date", new Date())
|
||||
println d.city +" "+d.region+" "+d.name +" 区域修改为:"+updateRegion
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
if(flag!="pass"){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("status", 1)
|
||||
udata.put("m_date", new Date())
|
||||
println d.city +" "+d.region+" "+d.name +" 状态修改为:删除"
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def findEsPoiByCity(city){
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
|
||||
def rets = []
|
||||
|
||||
String[] includes = ["city", "district", "name", "township", "status", "id"]
|
||||
def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi", queryBuilder,includes)
|
||||
for(def d: datas){
|
||||
def o = new BasicDBObject()
|
||||
o.put("city", d.city)
|
||||
o.put("district", d.district)
|
||||
o.put("name", d.name)
|
||||
o.put("township", d.township)
|
||||
o.put("location", d.location)
|
||||
o.put("status", d.status)
|
||||
o.put("id", d.id)
|
||||
rets << o
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
def poiHaveRegionProcess(city, set, rMap, datas){
|
||||
def results = []
|
||||
for(def d : datas){
|
||||
if(set.contains(d.district)){
|
||||
continue
|
||||
}
|
||||
def updateRegion = null
|
||||
for(def key : set){
|
||||
def alias = rMap.get(key)
|
||||
if(alias.contains(d.district)){
|
||||
updateRegion = key
|
||||
break;
|
||||
}
|
||||
}
|
||||
if(updateRegion){
|
||||
def m = [:]
|
||||
m.put(d.id, ["district":updateRegion])
|
||||
results << m
|
||||
if(results.size()%100==0){
|
||||
updateRegionByEs(results)
|
||||
results.clear()
|
||||
}
|
||||
println d.city +" "+d.district+" "+d.name +" "+d.status+" 区域修改为:"+updateRegion
|
||||
|
||||
}else{
|
||||
def m = [:]
|
||||
m.put(d.id, ["status":1])
|
||||
results << m
|
||||
if(results.size()%100==0){
|
||||
updateRegionByEs(results)
|
||||
results.clear()
|
||||
}
|
||||
|
||||
println d.city +" "+d.district+" "+d.name +" "+d.status+" 状态修改为:删除"
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
if(results.size()>0){
|
||||
updateRegionByEs(results)
|
||||
results.clear()
|
||||
}
|
||||
println city +" 处理完成"
|
||||
}
|
||||
|
||||
|
||||
|
||||
def updateRegionByEs(rows){
|
||||
def client = ElasticUtils89.getClient("192.168.0.89")
|
||||
def bulkRequest = new BulkRequest()
|
||||
|
||||
rows.each { row ->
|
||||
def id = row.keySet().iterator().next()
|
||||
def doc = row.get(id)
|
||||
|
||||
def updateRequest = new UpdateRequest("didata_map_poi", id)
|
||||
updateRequest.doc(doc, XContentType.JSON)
|
||||
bulkRequest.add(updateRequest)
|
||||
}
|
||||
|
||||
try {
|
||||
// 执行批量更新请求
|
||||
def bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT)
|
||||
|
||||
// 检查响应
|
||||
if (bulkResponse.hasFailures()) {
|
||||
println("批量更新中有失败的操作:")
|
||||
bulkResponse.buildFailureMessage().each { println(it) }
|
||||
} else {
|
||||
println("批量更新成功!")
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
// 关闭客户端
|
||||
try {
|
||||
client.close()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// def queryPoiDatasByCity(){
|
||||
// def esUrl = "http://localhost:9200/your_index/_search"
|
||||
// def pageSize = 1000 // 每页返回的记录数
|
||||
// def scrollSize = '1m' // scroll上下文的生存时间
|
||||
//
|
||||
//// 初始化HttpClient和ObjectMapper
|
||||
// CloseableHttpClient httpClient = HttpClients.createDefault()
|
||||
// ObjectMapper objectMapper = new ObjectMapper()
|
||||
//
|
||||
//// 初始化scroll请求
|
||||
// def scrollRequest = """
|
||||
//{
|
||||
// "size": $pageSize,
|
||||
// "query": {
|
||||
// "match_all": {}
|
||||
// },
|
||||
// "scroll": "$scrollSize"
|
||||
//}
|
||||
//"""
|
||||
//
|
||||
//// 发送scroll初始化请求
|
||||
// HttpGet scrollInitRequest = new HttpGet(esUrl)
|
||||
// scrollInitRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequest, "UTF-8"))
|
||||
// scrollInitRequest.setHeader("Content-Type", "application/json")
|
||||
//
|
||||
// HttpResponse scrollInitResponse = httpClient.execute(scrollInitRequest)
|
||||
// HttpEntity entity = scrollInitResponse.getEntity()
|
||||
// String scrollId = objectMapper.readValue(EntityUtils.toString(entity), Map.class).get("_scroll_id").toString()
|
||||
//
|
||||
//// 存储所有结果
|
||||
// def allResults = []
|
||||
//
|
||||
//// 发送scroll请求获取数据,直到没有更多数据为止
|
||||
// while (true) {
|
||||
// def scrollRequestWithId = """
|
||||
// {
|
||||
// "scroll_id": "$scrollId",
|
||||
// "scroll": "$scrollSize"
|
||||
// }
|
||||
// """
|
||||
//
|
||||
// HttpGet scrollGetRequest = new HttpGet(esUrl + "?scroll")
|
||||
// scrollGetRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequestWithId, "UTF-8"))
|
||||
// scrollGetRequest.setHeader("Content-Type", "application/json")
|
||||
//
|
||||
// HttpResponse scrollGetResponse = httpClient.execute(scrollGetRequest)
|
||||
// entity = scrollGetResponse.getEntity()
|
||||
// def responseMap = objectMapper.readValue(EntityUtils.toString(entity), Map.class)
|
||||
//
|
||||
// def hits = responseMap.get("hits").get("hits")
|
||||
// if (hits.isEmpty()) {
|
||||
// break // 没有更多数据,跳出循环
|
||||
// }
|
||||
//
|
||||
// // 合并当前页的数据
|
||||
// allResults.addAll(hits.map({ it.get("_source") }))
|
||||
//
|
||||
// // 更新scrollId以进行下一次请求
|
||||
// scrollId = responseMap.get("_scroll_id").toString()
|
||||
// }
|
||||
//
|
||||
// // 关闭HttpClient
|
||||
// httpClient.close()
|
||||
//
|
||||
// // 输出或处理全量数据
|
||||
// println(allResults)
|
||||
//
|
||||
// // 注意:实际应用中,你可能需要在获取完数据后清除scroll上下文,以释放资源。
|
||||
// }
|
||||
|
||||
|
||||
}
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
//更新第三方均价表中小区区域
|
||||
class RegionUpdateBatch {
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
static main(args) {
|
||||
def task = new RegionUpdateBatch()
|
||||
task.run()
|
||||
|
||||
}
|
||||
def run(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
for(def city : citys){
|
||||
def districtMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["city","region", "name"])
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city", 1)
|
||||
keys.put("region", 1)
|
||||
keys.put("name", 1)
|
||||
keys.put("seaweed_id", 1)
|
||||
def statistics = AVERAGE_STATISTICS.find(new BasicDBObject("city", city), keys)
|
||||
def total = statistics.size()
|
||||
final CountDownLatch latch = new CountDownLatch(total)
|
||||
def totalCount = new AtomicLong()
|
||||
for(def i=0;i<statistics.size();i++){
|
||||
final def d = statistics[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def dist = districtMap.get(d.seaweed_id)
|
||||
if(dist && dist.region!=d.region){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("region", dist.region)
|
||||
AVERAGE_STATISTICS.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata),false,false)
|
||||
}
|
||||
println totalCount.get() + " / "+total + " "+ d.city
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
+9
-3
@@ -35,7 +35,7 @@ class ThirdPriceMappingBatch extends BaseBatch{
|
||||
|
||||
final DBCollection dataProcessColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process", "192.168.0.235", 27018)
|
||||
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
static main(args) {
|
||||
runBatch(new ThirdPriceMappingBatch(), args);
|
||||
@@ -48,7 +48,7 @@ class ThirdPriceMappingBatch extends BaseBatch{
|
||||
// //½Ó¿Ú½øÐÐÆ¥Åä
|
||||
match()
|
||||
|
||||
// addOldMatchResult()
|
||||
addOldMatchResult()
|
||||
}
|
||||
|
||||
def match(){
|
||||
@@ -119,7 +119,12 @@ class ThirdPriceMappingBatch extends BaseBatch{
|
||||
def addNemCommunity(){
|
||||
def citys = CommonUtils.findAllCity()
|
||||
def total = 0
|
||||
def index = 0
|
||||
for(def c : citys){
|
||||
index ++
|
||||
if(index < 70){
|
||||
continue
|
||||
}
|
||||
def crawlDatas = thirdPartyPriceColl.find(new BasicDBObject("city", c).append("s_date", curDate)).toList()
|
||||
def addCount = addDatasToDB(crawlDatas)
|
||||
println c + " ÐÂÔö£º"+ addCount
|
||||
@@ -151,8 +156,9 @@ class ThirdPriceMappingBatch extends BaseBatch{
|
||||
sd.each {k,v->
|
||||
udata.put(k,v)
|
||||
}
|
||||
udata.removeField("_id")
|
||||
udata.put("m_date", new Date())
|
||||
thirdSeaweedMappingColl.update(query, new BasicDBObject('$set', udata))
|
||||
thirdSeaweedMappingColl.update(query, new BasicDBObject('$set', udata),false, false)
|
||||
}else{
|
||||
sd.put("m_date", new Date())
|
||||
thirdSeaweedMappingColl.save(sd)
|
||||
|
||||
+111
@@ -0,0 +1,111 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
/**
|
||||
* 乡镇街道 判断 中心点是否在围栏或一个围栏是否包含多个中心点
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class TownCenterBatch extends BaseBatch{
|
||||
|
||||
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new TownCenterBatch(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def city = "上海市"
|
||||
def shapes = findHasShapeTown(city)
|
||||
def centers = findAllCenters(city)
|
||||
def retMap = checkCenter(shapes, centers)
|
||||
retMap.each {key,values->
|
||||
if(values.size()>1){
|
||||
println values
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def checkCenter(shapes, centers){
|
||||
def map = [:]
|
||||
def index = 0
|
||||
for(def town : shapes){
|
||||
index ++
|
||||
def points = InsidePolygon.createPoint4(town.shape.coordinates)
|
||||
//1.先判断当前镇的中心点是否在围栏中
|
||||
if(town.location){
|
||||
def flag = InsidePolygon.isInArea(town.location.lon, town.location.lat, points)
|
||||
if(!flag){
|
||||
println town.city + " "+town.district+" "+town.street+" 不在本身围栏中"
|
||||
}else{
|
||||
def list = map.get(town.city + "#"+town.district+"#"+town.street)
|
||||
if(!list){
|
||||
list = []
|
||||
map.put(town.city + "#"+town.district+"#"+town.street, list)
|
||||
}
|
||||
list << town.city + "#"+town.district+"#"+town.street
|
||||
}
|
||||
}
|
||||
//2.判断是否有其它点在围栏中
|
||||
for(def center : centers){
|
||||
if(center._id.toString() != town._id.toString()){
|
||||
def flag = InsidePolygon.isInArea(center.location.lon, center.location.lat, points)
|
||||
if(flag){
|
||||
def list = map.get(town.city + "#"+town.district+"#"+town.street)
|
||||
if(!list){
|
||||
list = []
|
||||
map.put(town.city + "#"+town.district+"#"+town.street, list)
|
||||
}
|
||||
list << center.city + "#"+center.district+"#"+center.street
|
||||
println center.city + " "+center.district+" "+center.street+" 在 "+town.city + " "+town.district+" "+town.street+" 围栏中"
|
||||
}
|
||||
}
|
||||
}
|
||||
def list = map.get(town.city + "#"+town.district+"#"+town.street)
|
||||
if(list){
|
||||
println index+" "+town.city + "#"+town.district+"#"+town.street + " "+list.size()
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
def findHasShapeTown(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("type", "乡镇街道")
|
||||
q.put("shape", new BasicDBObject('$exists', true))
|
||||
if(city){
|
||||
q.put("city", city)
|
||||
}
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("location", 1)
|
||||
keys.put("city", 1)
|
||||
keys.put("district", 1)
|
||||
keys.put("street", 1)
|
||||
keys.put("shape", 1)
|
||||
return ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).toList()
|
||||
}
|
||||
|
||||
def findAllCenters(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("type", "乡镇街道")
|
||||
q.put("location", new BasicDBObject('$exists', true))
|
||||
if(city){
|
||||
q.put("city", city)
|
||||
}
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("location", 1)
|
||||
keys.put("city", 1)
|
||||
keys.put("district", 1)
|
||||
keys.put("street", 1)
|
||||
return ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).toList()
|
||||
}
|
||||
}
|
||||
+136
-31
@@ -1,6 +1,7 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.GeocodeUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
@@ -77,7 +78,7 @@ class TownMiddleProcess {
|
||||
//对比中间表的街道与community街道是否一致,因为元极反馈类似“高密市醴泉街道”解析有问题,要去掉“高密市”,然后去了中间表
|
||||
// updateCommunityDiffStreet()
|
||||
// updateTownPoline()//更新乡镇街道围栏
|
||||
updateTownPolineBatch()//批量更新乡镇街道围栏
|
||||
// updateTownPolineBatch()//批量更新乡镇街道围栏
|
||||
// checkTown()
|
||||
//区域民政部名称更新到乡镇街道中
|
||||
// updateTownshipByRegion()
|
||||
@@ -93,6 +94,8 @@ class TownMiddleProcess {
|
||||
|
||||
// addTownCenter()//百度接口添加中心点
|
||||
// checkCenterInTownRang()//检测乡镇街道中心点是否在围栏中
|
||||
|
||||
addSpecialCityTownAlias()
|
||||
}
|
||||
|
||||
|
||||
@@ -900,41 +903,92 @@ class TownMiddleProcess {
|
||||
}
|
||||
|
||||
def updateTownPolineBatch(){
|
||||
def i = 0
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\乡镇街道围栏开发区数据.xlsx",{ row->
|
||||
if(row['R'] && row['R']=="新增围栏"){
|
||||
def q = new BasicDBObject("city", row['B']).append("region", row['C']).append("town",row['D'])
|
||||
println row['B']+" "+ row['D']+" "+TOWNPOLYLINE_COLL.count(q)
|
||||
def d = TOWNPOLYLINE_COLL.findOne(q)
|
||||
if(d){
|
||||
def resultMap = [:]
|
||||
resultMap.type = "polygon"
|
||||
def arrs = d.polyline.split("\\|")
|
||||
def rets = []
|
||||
for(def s : arrs){
|
||||
def points = s.split(";")
|
||||
def pointList = []
|
||||
for(def pointStr : points){
|
||||
def pointArr = pointStr.split(",")
|
||||
def lon = DataHandleUtils.getDoubleRound(6, pointArr[0].toString().toDouble())
|
||||
def lat = DataHandleUtils.getDoubleRound(6, pointArr[1].toString().toDouble())
|
||||
pointList << [lon, lat]
|
||||
i++
|
||||
if(i>=134){
|
||||
def q = new BasicDBObject("city", row['B']).append("region", row['C']).append("town",row['D'])
|
||||
println row['B']+" "+ row['D']+" "+TOWNPOLYLINE_COLL.count(q)
|
||||
def d = TOWNPOLYLINE_COLL.findOne(q)
|
||||
if(d){
|
||||
def resultMap = [:]
|
||||
def bd_resultMap = [:]
|
||||
|
||||
resultMap.type = "polygon"
|
||||
bd_resultMap.type = "polygon"
|
||||
|
||||
def arrs = d.polyline.split("\\|")
|
||||
def rets = []
|
||||
def bd_rets = []
|
||||
|
||||
println row['N'] +" 共有:"+arrs.size()+" 个围栏"
|
||||
def index = 1
|
||||
for(def s : arrs){
|
||||
def points = s.split(";")
|
||||
def pointList = []
|
||||
def bd_pointList = []
|
||||
|
||||
for(def pointStr : points){
|
||||
def pointArr = pointStr.split(",")
|
||||
def lon = DataHandleUtils.getDoubleRound(6, pointArr[0].toString().toDouble())
|
||||
def lat = DataHandleUtils.getDoubleRound(6, pointArr[1].toString().toDouble())
|
||||
pointList << [lon, lat]
|
||||
|
||||
// 高德转百度
|
||||
def baidu = BaiDuApiUtils.geoconv(lon, lat, 3)
|
||||
def bd_lng = baidu.x
|
||||
def bd_lat = baidu.y
|
||||
bd_pointList << [bd_lng, bd_lat]
|
||||
}
|
||||
|
||||
if(index<=1){
|
||||
resultMap.coordinates = pointList
|
||||
bd_resultMap.coordinates = bd_pointList
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("shape", resultMap)
|
||||
udata.put("operator", "高娟")
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("bd_shape",bd_resultMap)
|
||||
|
||||
def updateQuery = new BasicDBObject("name",row['N'])
|
||||
DEVELOPMENT_ZONE_43_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||
}else{
|
||||
def dbo = DEVELOPMENT_ZONE_43_COLL.findOne(new BasicDBObject("name",row['N']))
|
||||
def obj = new BasicDBObject()
|
||||
dbo.each {k,v->
|
||||
if(k!='_id'){
|
||||
obj.put(k,v)
|
||||
}
|
||||
}
|
||||
|
||||
resultMap.coordinates = pointList
|
||||
bd_resultMap.coordinates = bd_pointList
|
||||
obj.put("shape", resultMap)
|
||||
obj.put("operator", "高娟")
|
||||
obj.put("m_date", new Date())
|
||||
obj.put("bd_shape",bd_resultMap)
|
||||
obj.put("create_time", new Date())
|
||||
obj.put("block", dbo.name+"-地块"+index)
|
||||
DEVELOPMENT_ZONE_43_COLL.save(obj)
|
||||
}
|
||||
|
||||
// rets << pointList
|
||||
// bd_rets << bd_pointList
|
||||
index++
|
||||
}
|
||||
rets << pointList
|
||||
|
||||
|
||||
// if(rets.size()>1) {
|
||||
// resultMap.type = "MultiPolygon"
|
||||
// bd_resultMap.type = "MultiPolygon"
|
||||
// }
|
||||
// println d.city+" "+d.region+" "+d.town+" "+rets.size()
|
||||
row['S'] = "已完成"
|
||||
}
|
||||
resultMap.coordinates = rets
|
||||
if(rets.size()>1) {
|
||||
resultMap.type = "MultiPolygon"
|
||||
}
|
||||
println d.city+" "+d.region+" "+d.town+" "+rets.size()
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("shape", resultMap)
|
||||
|
||||
|
||||
def updateQuery = new BasicDBObject("name",row['N'])
|
||||
DEVELOPMENT_ZONE_43_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||
row['S'] = "已完成"
|
||||
}
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1249,6 +1303,57 @@ class TownMiddleProcess {
|
||||
for(def d : dbcursor){
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def addSpecialCityTownAlias(){
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
|
||||
def citys = ["东莞","中山","嘉峪关","昆玉","石河子","阿拉尔","图木舒克","五家渠","济源","仙桃","潜江","天门","神农架","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中","北屯","铁门关","双河","可克达拉"]
|
||||
def regionMap = CommonDatautils.getAllCityFullRegions()
|
||||
for(def city : citys){
|
||||
def townMap = regionMap.get(city)
|
||||
def streets = findAllStreetByCity(city)
|
||||
for(def s : streets){
|
||||
def set = townMap.keySet()
|
||||
def alias = null
|
||||
for(def name : set){
|
||||
def townObj = townMap.get(name)
|
||||
if(townObj && townObj.pattern && townObj.pattern.contains(s.street)){
|
||||
alias = townObj.pattern
|
||||
break;
|
||||
}
|
||||
}
|
||||
if(alias){
|
||||
def newAlias = []
|
||||
for(def n : alias){
|
||||
if(n!=s.street){
|
||||
newAlias << n
|
||||
}
|
||||
}
|
||||
if(alias){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("alias", newAlias)
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id", s._id), new BasicDBObject('$set', udata),false, false)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def findAllStreetByCity(city){
|
||||
def data = fullCityMap.get(city)
|
||||
def fullCityName = null
|
||||
if(data){
|
||||
fullCityName = data.city
|
||||
}
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("type", "乡镇街道")
|
||||
q.put("city", fullCityName)
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("street", 1)
|
||||
return ADMINISTRATIVE_MAPPING_COLL.find(q, keys).toList()
|
||||
}
|
||||
}
|
||||
|
||||
+279
@@ -0,0 +1,279 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.update.UpdateRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
/**
|
||||
* 区域标准化
|
||||
* 1.小区库乡镇街道标准化,包含小区,商铺,写字楼
|
||||
* 2.poi数据库中乡镇街道标准化
|
||||
*/
|
||||
class TownStandardBatch extends BaseBatch{
|
||||
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
|
||||
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
|
||||
|
||||
def poiEsUrl = "http://192.168.0.89:9200/didata_map_poi/_search"
|
||||
|
||||
def elasticsearchUrl = "http://192.168.0.89:9200/_search/scroll"
|
||||
def initialSearchUrl = "http://192.168.0.89:9200/didata_map_poi/_search?scroll=1m"
|
||||
def scrollTime = "1m" // Scroll context time-to-live
|
||||
def pageSize = 1000 // Number of hits to return per scroll
|
||||
|
||||
static main(args) {
|
||||
runBatch(new TownStandardBatch(), args)
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
TownStandard()
|
||||
}
|
||||
|
||||
def TownStandard(){
|
||||
// townStandardByCat("office")
|
||||
townStandardByPoi()
|
||||
}
|
||||
def townStandardByCat(def cat){
|
||||
//"三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德",
|
||||
def citys = ["重庆","龙岩","南平","漳州","福州","厦门","泉州","莆田","三明","宁德"]//CommonDatautils.findAllCity()
|
||||
for(def city : citys){
|
||||
|
||||
def data = fullCityMap.get(city)
|
||||
def fullCityName = null
|
||||
if(data){
|
||||
fullCityName = data.city
|
||||
}
|
||||
|
||||
def mzbRegionMap = CommonDatautils.queryMzbRegion(fullCityName)
|
||||
def shortRegionNameMap = CommonDatautils.getRegionShortName(fullCityName)
|
||||
def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district", "street", "alias"], mzbRegionMap)
|
||||
if(townMap){
|
||||
def regionSet = townMap.keySet()
|
||||
if(regionSet.size()>1){
|
||||
haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap)
|
||||
}else{
|
||||
println city +" 无区域"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def townStandardByPoi(){
|
||||
def citys = citys = ["重庆","龙岩","南平","漳州","福州","厦门","泉州","莆田","三明","宁德"]//CommonDatautils.findAllCity()
|
||||
for(def city : citys){
|
||||
|
||||
def data = fullCityMap.get(city)
|
||||
def fullCityName = null
|
||||
if(data){
|
||||
fullCityName = data.city
|
||||
}
|
||||
if(fullCityName){
|
||||
def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district", "street", "alias"])
|
||||
def rMap = regionFullNameMap.get(fullCityName)
|
||||
if(rMap){
|
||||
def set = rMap.keySet()
|
||||
def rets = findEsPoiByCity(fullCityName)
|
||||
println city +" "+rets.size()
|
||||
if(set.size()>1){
|
||||
poiHaveRegionProcess(fullCityName, townMap, rets)
|
||||
}else{
|
||||
// haveStreetProcess(city, set, cat)
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println "没找到相应的全称:"+city
|
||||
}
|
||||
}
|
||||
}
|
||||
def haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap){
|
||||
def results = []
|
||||
if(fullCityName){
|
||||
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name", "township"])
|
||||
for(def d : districts){
|
||||
if(d.township){
|
||||
def towns = null
|
||||
def updateTown = null
|
||||
for(def m : townMap){
|
||||
def fullRegionName = shortRegionNameMap.get(d.region)
|
||||
if(m.key == fullRegionName){
|
||||
towns = m.value
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
def opt = true
|
||||
if(towns){
|
||||
for(def town : towns){
|
||||
def townAlias = []
|
||||
if(town.alias){
|
||||
townAlias = town.alias
|
||||
}
|
||||
townAlias << town.street
|
||||
if(townAlias.contains(d.township)){
|
||||
if(d.township!=town.street){
|
||||
updateTown = town.street
|
||||
}else{
|
||||
opt = false
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if(opt){
|
||||
if(updateTown){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("township", updateTown)
|
||||
udata.put("m_date", new Date())
|
||||
println d.city +" "+d.region+" "+d.name+" "+d.township +" 乡镇修改为:"+updateTown
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("township", "")
|
||||
udata.put("m_date", new Date())
|
||||
println d.city +" "+d.region+" "+d.name+" "+d.township +" 乡镇修改为:空值"
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def poiHaveRegionProcess(fullCityName, townMap, rets){
|
||||
def results = []
|
||||
if(fullCityName){
|
||||
for(def d : rets){
|
||||
if(d.township){
|
||||
def towns = null
|
||||
def updateTown = null
|
||||
for(def m : townMap){
|
||||
def townDatas = townMap.get(d.district)
|
||||
if(townDatas){
|
||||
towns = townDatas
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
def opt = true
|
||||
if(towns){
|
||||
for(def town : towns){
|
||||
def townAlias = []
|
||||
if(town.alias){
|
||||
townAlias = town.alias
|
||||
}
|
||||
townAlias << town.street
|
||||
if(townAlias.contains(d.township)){
|
||||
if(d.township!=town.street){
|
||||
updateTown = town.street
|
||||
}else{
|
||||
opt = false
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if(opt){
|
||||
if(updateTown){
|
||||
def m = [:]
|
||||
m.put(d.id, ["township":updateTown])
|
||||
results << m
|
||||
// if(results.size()%100==0){
|
||||
// updateByEs(results)
|
||||
// results.clear()
|
||||
// }
|
||||
println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown
|
||||
}else{
|
||||
def m = [:]
|
||||
m.put(d.id, ["township":""])
|
||||
results << m
|
||||
// if(results.size()%100==0){
|
||||
// updateByEs(results)
|
||||
// results.clear()
|
||||
// }
|
||||
println d.city +" "+d.district+" "+d.name +" "+d.township+" 状态修改为:空值"
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(results.size()>0){
|
||||
updateByEs(results)
|
||||
results.clear()
|
||||
}
|
||||
println fullCityName +" 处理完成"
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def updateByEs(rows){
|
||||
def client = ElasticUtils89.getClient("192.168.0.89")
|
||||
def bulkRequest = new BulkRequest()
|
||||
|
||||
rows.each { row ->
|
||||
def id = row.keySet().iterator().next()
|
||||
def doc = row.get(id)
|
||||
|
||||
def updateRequest = new UpdateRequest("didata_map_poi", id)
|
||||
updateRequest.doc(doc, XContentType.JSON)
|
||||
bulkRequest.add(updateRequest)
|
||||
}
|
||||
|
||||
try {
|
||||
// 执行批量更新请求
|
||||
def bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT)
|
||||
|
||||
// 检查响应
|
||||
if (bulkResponse.hasFailures()) {
|
||||
println("批量更新中有失败的操作:")
|
||||
bulkResponse.buildFailureMessage().each { println(it) }
|
||||
} else {
|
||||
println("批量更新成功!")
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
// 关闭客户端
|
||||
try {
|
||||
client.close()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def findEsPoiByCity(city){
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("status", 0))
|
||||
def rets = []
|
||||
|
||||
String[] includes = ["city", "district", "name", "township", "status", "id"]
|
||||
def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi", queryBuilder, includes)
|
||||
for(def d: datas){
|
||||
def o = new BasicDBObject()
|
||||
o.put("city", d.city)
|
||||
o.put("district", d.district)
|
||||
o.put("name", d.name)
|
||||
o.put("township", d.township)
|
||||
o.put("status", d.status)
|
||||
o.put("id", d.id)
|
||||
rets << o
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+12
-12
@@ -42,7 +42,7 @@ class TownshipStrandUpdate {
|
||||
// task.updateTown()
|
||||
// task.stat()
|
||||
|
||||
// task.updateDiffRegionTown()
|
||||
task.updateDiffRegionTown()
|
||||
// task.cleanTown()
|
||||
// task.checkTownFromMongo()
|
||||
// task.updateTownMapping()
|
||||
@@ -423,29 +423,29 @@ class TownshipStrandUpdate {
|
||||
}
|
||||
|
||||
def updateDiffRegionTown(){
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\shop-不同区域.xlsx"
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
def lists = CommonDatautils.findDistrictList(d['B'], null, d['D'], "shop")
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\district-不同区域.xlsx"
|
||||
ExcelUtils.modify(fileName,{d->
|
||||
def lists = CommonDatautils.findDistrictList(d['A'], null, d['C'], "district")
|
||||
d['AA'] = lists.size()
|
||||
//只处理库中只有一个的
|
||||
if(lists && lists.size()==1 && d['Z']=="包含" && !d['AB']){
|
||||
def newRegion = RegionData.formatRegion(d['B'], d['M'])
|
||||
if(lists && lists.size()==1 && d['R']){
|
||||
def newRegion = RegionData.formatRegion(d['A'], d['R'])
|
||||
if(newRegion){
|
||||
if(d['X'] && d['X'].toString().contains(d['M']+" "+d['N'])){ //地图街道在中间表中
|
||||
// if(d['X'] && d['X'].toString().contains(d['M']+" "+d['N'])){ //地图街道在中间表中
|
||||
def dbo = lists[0]
|
||||
def arrs = d['X'].split(" ")
|
||||
// def arrs = d['X'].split(" ")
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("region", newRegion)
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("township", arrs[1])
|
||||
// udata.put("township", arrs[1])
|
||||
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata))
|
||||
CommonDatautils.saveOptLog(dbo.city, dbo.region, dbo.name, "shop", "update","高娟")
|
||||
CommonDatautils.saveOptLog(dbo.city, dbo.region, dbo.name, "district", "update-region","高娟")
|
||||
d['AB'] = "已处理"
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
})
|
||||
},1)
|
||||
}
|
||||
|
||||
def cleanTown(){
|
||||
|
||||
+16
-2
@@ -47,7 +47,7 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
//"alias":"alias",
|
||||
//ownership_type:权属类型 为商品房、经适房、动迁房等
|
||||
//house_property 产权性质:国有产权 "house_property":"网站没有",
|
||||
def fieldsMap = ["estate_charge":"property_fee","block":"block","build_date":"build_year","estate_company":"property_company","developer":"developer",
|
||||
def fieldsMap = ["lng":"location","estate_charge":"property_fee","block":"block","build_date":"build_year","estate_company":"property_company","developer":"developer",
|
||||
"virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area","household_count":"households",
|
||||
"car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address","build_structure":"building_structure",
|
||||
"build_type":"build_type", "land_property":"ownership_type","live_date":"live_date",
|
||||
@@ -115,7 +115,7 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
println "【通过匹配填充字段】网站:"+seaweedmapSource +" Es:"+esRets.size()+" 融合小区占比:"+DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/totalCount)+"%"
|
||||
def precentMap = fieldUpdate(totalCount, updateRets)
|
||||
println c +" "+precentMap.toString()
|
||||
//updateDB(c, updateRets)
|
||||
updateDB(c, updateRets)
|
||||
println "------------------------------------------------------------------------------"
|
||||
def pre1 = 0
|
||||
def pre2 = 0
|
||||
@@ -212,6 +212,11 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
def rets = []
|
||||
rets << ["第三方城市","第三方区域","第三方名称","第三方地址","第三方户数","匹配区域","匹配名称","匹配可信度","方式"]
|
||||
for(def matchDist : matchRets){
|
||||
|
||||
// if(!["智富中心","龙山新寓","天怡瀚园","远盛凤林府","京口路小区","梦溪路小区","浩创优尚","溪山丽苑","旭园"].contains(matchDist.name)){
|
||||
// continue
|
||||
// }
|
||||
|
||||
def dist = CommonDatautils.findDistrictFieldById(matchDist.match_id, fields)
|
||||
if(!dist){
|
||||
println c+" "+matchDist.match_id+" 不存在"
|
||||
@@ -306,6 +311,15 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
if(newVal){
|
||||
newValues.put(seaweedfield, newVal)
|
||||
}
|
||||
if(seaweedfield == "lng"){
|
||||
def locationMap = FiledStrand.processLngAndLat(esVal)
|
||||
if(locationMap){
|
||||
newValues.put("lat", locationMap.lat)
|
||||
newValues.put("lng", locationMap.lng)
|
||||
newValues.put("lat2", locationMap.lat2)
|
||||
newValues.put("lng2", locationMap.lng2)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+56
@@ -0,0 +1,56 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
/**
|
||||
* 临时移除 相同名称即在正常状态下有,在删除状态下也有的小区
|
||||
*/
|
||||
class RemoveDistrict {
|
||||
|
||||
public static void main(String[] args) {
|
||||
def task = new RemoveDistrict()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def citys = CommonUtils.findAllCity()
|
||||
def cat = "shop"
|
||||
def keys = []
|
||||
citys.each {c->
|
||||
def districts = CommonDatautils.findAllDistricts(c, cat, ["city", "region", "name", "status"])
|
||||
districts.each {it->
|
||||
def key = it.city +"##"+it.region+"##"+it.name
|
||||
if(!keys.contains(key)){
|
||||
keys << key
|
||||
}else{
|
||||
println it.toString()+" 删除"
|
||||
DbUtils.seaweed.remove(new BasicDBObject("_id",it._id))
|
||||
}
|
||||
// def list = CommonDatautils.findAllStatusDistrict(it.city, it.region, it.name, cat, ["city", "region", "name", "status"])
|
||||
// if(list && list.size()>1){
|
||||
// println it.toString() + " "+list.size()
|
||||
// for(def d : list){
|
||||
//// if(d.status.toString()=="0"){
|
||||
//// println d.toString()+" 保留"
|
||||
//// }else if(d.status.toString()=="1"){
|
||||
//// println d.toString()+" 删除"
|
||||
//// DbUtils.seaweed.remove(new BasicDBObject("_id",d._id))
|
||||
//// }
|
||||
//
|
||||
// if(d.status.toString()=="1"){
|
||||
// println d.toString()+" 保留"
|
||||
// }else if(d.status.toString()!="1"){
|
||||
// println d.toString()+" 删除"
|
||||
// DbUtils.seaweed.remove(new BasicDBObject("_id",d._id))
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// println "................................................................................................"
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+146
@@ -0,0 +1,146 @@
|
||||
package com.fangjia.script.gj.images
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 更新小区图片
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class AddDistImagesBatch extends BaseBatch{
|
||||
final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018)
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
public static final DBCollection seaweed_fs_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_images", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new AddDistImagesBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
// def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
|
||||
def mdate = DateUtils.getFirstDayOfMonth(new Date(), -1)
|
||||
def citys = CommonUtils.findAllCity()
|
||||
citys.each {city->
|
||||
def communitys = findAllEsData2(city, mdate)
|
||||
println "抓取城市: "+city +" 第三方小区有图片数:"+communitys.size()
|
||||
//第三方匹配小区
|
||||
matchCommunitys(communitys)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findAllEsData2(city, mdate){
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("m_date").from(mdate.getTime()).includeLower(true)
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("source", "安居客小程序"))
|
||||
.must(date_range).must(date_range)
|
||||
|
||||
def rets = []
|
||||
def datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "community", queryBuilder);
|
||||
if(datas) {
|
||||
for (int k = 0; k < datas.size(); k++) {
|
||||
def d = datas[k];
|
||||
if(d.house_type_images_2 || d.outdoor_images) {
|
||||
def data = new BasicDBObject()
|
||||
data.put("city", d.city)
|
||||
data.put("region", d.region)
|
||||
data.put("name", d.name)
|
||||
data.put("id", d.id)
|
||||
data.put("out_images", d.outdoor_images)
|
||||
data.put("house_type_images", d.house_type_images_2)
|
||||
rets << data
|
||||
}
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findAllEsData(city, mdate){
|
||||
def names = new HashSet()
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery("select * from community where source='安居客小程序' and city='${city}' and m_date>='${mdate}' order by m_date", { d->
|
||||
if(d.house_type_images_2 || d.outdoor_images){
|
||||
def s = d.city +"#"+d.region+"#"+d.name+"#"+d.property_type
|
||||
if(!names.contains(s)){
|
||||
names << s
|
||||
def data = new BasicDBObject()
|
||||
data.put("city", d.city)
|
||||
data.put("region", d.region)
|
||||
data.put("name", d.name)
|
||||
data.put("out_images", d.outdoor_images)
|
||||
data.put("house_type_images", d.house_type_images_2)
|
||||
rets << data
|
||||
}
|
||||
}
|
||||
})
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def matchCommunitys(communitys){
|
||||
def addCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(communitys.size())
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def total = communitys.size()
|
||||
for(def i=0;i<communitys.size();i++){
|
||||
final def d = communitys[i]
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
try{
|
||||
addCount.addAndGet(1)
|
||||
def query = new BasicDBObject("city", d.city).append("district",d.region).append("name",d.name).append("source","安居客").append("match_id", new BasicDBObject('$exists', true))
|
||||
def obj = thirdSeaweedMappingColl.findOne(query)
|
||||
if(obj) {
|
||||
def vo = new BasicDBObject()
|
||||
vo.put("city", obj.match_city)
|
||||
vo.put("district", obj.match_district)
|
||||
vo.put("name", obj.match_name)
|
||||
if(d.out_images){
|
||||
vo.put("out_pics", d.out_images)
|
||||
}
|
||||
if(d.house_type_images){
|
||||
vo.put("floorplan_pics", d.house_type_images)
|
||||
}
|
||||
vo.put("m_date", new Date())
|
||||
rets.add(vo)
|
||||
def q = new BasicDBObject("_id", new ObjectId(obj.match_id))
|
||||
seaweed_fs_coll.update(q, new BasicDBObject('$set', vo), true,false)
|
||||
println addCount.get()+"/"+total +" [匹配库中存在] "
|
||||
}else{
|
||||
println addCount.get()+"/"+total +" [没找到相应的匹配数据] "+d._id.toString()
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
// }
|
||||
// });
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
}
|
||||
+45
-12
@@ -2,6 +2,7 @@ package com.fangjia.script.gj.poc
|
||||
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
@@ -25,6 +26,10 @@ class AddPriceTest {
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
//235为房价网内部计算均价的库
|
||||
def pgsqlInstance235 = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
public static final DBCollection confirm_price_date = MongoUtils.getCollectionByName("fangjia", "confirm_price_data", "mongo.db.fangjia.com")
|
||||
|
||||
@@ -33,9 +38,9 @@ class AddPriceTest {
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def filePath = "D:\\数据\\建行\\房产估值\\20240716-20240815 _业务反馈价格.xlsx"
|
||||
// addGXDPrice(filePath)
|
||||
addCcbPrice(filePath)
|
||||
def filePath = "D:\\数据\\建行\\房产估值\\202410测试偏差较大案例统计数据test.xlsx"
|
||||
addGXDPrice(filePath)
|
||||
// addCcbPrice(filePath)
|
||||
// addThirdPrice(filePath)
|
||||
// addFjwPrice(filePath)
|
||||
// exportPkPrice()
|
||||
@@ -173,17 +178,22 @@ class AddPriceTest {
|
||||
def cityId = cityCodeDataMap.get(d['V'])
|
||||
def map = findAvgPrice(d['U'], cityId)
|
||||
if(map){
|
||||
d['F'] = map.avg_price
|
||||
// d['P'] = map.calc_type
|
||||
d['H'] = map.avg_price
|
||||
d['I'] = map.calc_type
|
||||
// d['AM'] = map.credit_value
|
||||
}
|
||||
|
||||
// def newPriceMap = findNewFjwAvgPrice(d['U'], cityId)
|
||||
// if(newPriceMap){
|
||||
// d['Q'] = newPriceMap.avg_price
|
||||
// }
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def findAvgPrice(def commid, def cityid){
|
||||
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202407 order by gujiabds_clc_yrmo_val desc limit 1"
|
||||
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202409 order by gujiabds_clc_yrmo_val desc limit 1"
|
||||
def map = [:]
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
map.avg_price = r.gujia_bds_clc_avprc
|
||||
@@ -202,6 +212,25 @@ class AddPriceTest {
|
||||
}
|
||||
|
||||
|
||||
def findNewFjwAvgPrice(def commid, def cityid){
|
||||
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from price_stat.avm_community_base_price where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202409 order by gujiabds_clc_yrmo_val desc limit 1"
|
||||
def map = [:]
|
||||
pgsqlInstance235.eachRow(sql, { r ->
|
||||
map.avg_price = r.gujia_bds_clc_avprc
|
||||
map.type = r.gujia_bds_clmtd
|
||||
map.month = r.gujiabds_clc_yrmo_val
|
||||
map.credit_value = r.gujia_bds_cfdc_dgr
|
||||
if(r.gujia_bds_clmtd == 1){
|
||||
map.calc_type = "人工估值"
|
||||
}else if(r.gujia_bds_clmtd == 2){
|
||||
map.calc_type = "样本计算"
|
||||
}else if(r.gujia_bds_clmtd == 3){
|
||||
map.calc_type = "神经网络"
|
||||
}
|
||||
})
|
||||
return map
|
||||
}
|
||||
|
||||
def addThirdPrice(filepath){
|
||||
ExcelUtils.multiThreadingModify(filepath, { d ->
|
||||
if (!d['X']) {
|
||||
@@ -241,12 +270,16 @@ class AddPriceTest {
|
||||
|
||||
def addGXDPrice(filepath){
|
||||
ExcelUtils.multiThreadingModify(filepath, { d ->
|
||||
if (!d['M']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['B'])
|
||||
if (!d['B']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
if(dist){
|
||||
def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
|
||||
// def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
|
||||
d['M'] = gxdPrice
|
||||
if(gxdPrice){
|
||||
d['B'] = gxdPrice.price
|
||||
d['C'] = DateUtils.formatDateByPattern(gxdPrice.c_date, "yyyy-MM-dd")
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -260,11 +293,11 @@ class AddPriceTest {
|
||||
q.put("district_name", name)
|
||||
q.put("source", "GXD")
|
||||
def dbcursor = confirm_price_date.find(q).sort(new BasicDBObject("c_date", -1)).limit(1)
|
||||
def price = null
|
||||
def obj = null
|
||||
for(def d : dbcursor){
|
||||
price = d.price
|
||||
obj = d
|
||||
}
|
||||
return price
|
||||
return obj
|
||||
}
|
||||
|
||||
|
||||
|
||||
+9
-2
@@ -26,6 +26,7 @@ import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.search.Scroll
|
||||
import org.elasticsearch.search.SearchHit
|
||||
import org.elasticsearch.search.builder.SearchSourceBuilder
|
||||
import org.elasticsearch.search.fetch.subphase.FetchSourceContext
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
@@ -35,7 +36,6 @@ import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.function.BiConsumer
|
||||
|
||||
/**
|
||||
* ¹¦ÄÜ˵Ã÷:Elasticsearch¹¤¾ßÀà
|
||||
*/
|
||||
@@ -228,14 +228,21 @@ class ElasticUtils89 {
|
||||
}
|
||||
|
||||
|
||||
static def executeSimpleQuery(String host, String indexName, QueryBuilder qb){
|
||||
static def executeSimpleQuery(String host, String indexName, QueryBuilder qb, includes){
|
||||
|
||||
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
|
||||
RestHighLevelClient client = getClient(host)
|
||||
SearchRequest searchRequest = new SearchRequest(indexName);
|
||||
searchRequest.scroll(scroll);
|
||||
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||
if(includes){
|
||||
String[] excludes = [] // 没有要排除的字段
|
||||
FetchSourceContext fetchSourceContext = new FetchSourceContext(true, includes, excludes)
|
||||
searchSourceBuilder.fetchSource(fetchSourceContext)
|
||||
}
|
||||
|
||||
searchSourceBuilder.query(qb);
|
||||
// searchSourceBuilder.size(1000)
|
||||
searchRequest.source(searchSourceBuilder);
|
||||
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
|
||||
String scrollId = searchResponse.getScrollId();
|
||||
|
||||
+29
-6
@@ -12,15 +12,13 @@ import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.AMQP
|
||||
|
||||
/**
|
||||
* 厦门涉台园区数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class RelatedTaiWanBatch extends BaseBatch{
|
||||
|
||||
def curDate = Integer.parseInt(DateUtils.formatDateByPattern(new Date(), "yyyyMM"));
|
||||
def curDate = Integer.parseInt(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"));
|
||||
|
||||
DBCollection yuanqu_qiye_coll = MongoUtils.getCollectionByName("crawl", "qzd_shetai_company_detail", "192.168.0.206", 27020)
|
||||
def relateTaiwn_yuanqu_datas = ["福州台商投资区","杏林台商投资区","集美台商投资区","海沧台商投资区","漳州台商投资区","泉州台商投资区","漳浦台湾农民创业园","漳平台湾农民创业园","仙游台湾农民创业园","清流台湾农民创业园","福清台湾农民创业园","惠安台湾农民创业园","海峡两岸集成电路产业合作试验区(厦门、泉州产业园)","海峡两岸生技和医疗健康产业合作区","海峡两岸农业合作试验区","海峡两岸(三明)现代林业合作实验区","漳州古雷石化基地","永泰闽台农业融合发展产业园","上杭闽台农业融合发展产业园","同安闽台农业融合发展产业园","南靖闽台农业融合发展产业园","漳浦闽台农业融合发展产业园","永春闽台农业融合发展产业园","永安闽台农业融合发展产业园","邵武闽台农业融合发展产业园","平潭闽台农业融合发展产业园","南靖闽台精密机械产业园"]
|
||||
@@ -36,9 +34,17 @@ class RelatedTaiWanBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
// createDatas() //企业库获取涉台园区企业
|
||||
updateRangeByPoint() //更改企业所在自贸区
|
||||
export() //导出数据
|
||||
|
||||
def day = new Date().getDate()
|
||||
if(day == 10){
|
||||
//createDatas() //企业库获取涉台园区企业
|
||||
updateRangeByPoint() //更改企业所在自贸区
|
||||
export() //导出数据
|
||||
}else{
|
||||
exportEmptyFile()
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -158,4 +164,21 @@ class RelatedTaiWanBatch extends BaseBatch{
|
||||
FileUtils.batchCreateFileByCustomSuffix([rdy_datas], fileName2, null, "utf-8", false, "","|+|");
|
||||
XmBankFtpUtils.ftpUpload(fileName2, curDate)
|
||||
}
|
||||
|
||||
def exportEmptyFile(){
|
||||
def fileName = "/home/goojia/xmbank/free_trade_zone_${curDate}_init.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\厦门\\涉台园区\\free_trade_zone_${curDate}_init.dat"
|
||||
}
|
||||
FileUtils.batchCreateFileByCustomSuffix([], fileName, null, "utf-8", false, "|@|","|+|");
|
||||
XmBankFtpUtils.ftpUpload(fileName, curDate)
|
||||
|
||||
def fileName2 = "/home/goojia/xmbank/free_trade_zone_${curDate}_init.dat.rdy"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName2 = "D:\\数据\\厦门\\涉台园区\\free_trade_zone_${curDate}_init.dat.rdy"
|
||||
}
|
||||
def rdy_datas = ["0"]
|
||||
FileUtils.batchCreateFileByCustomSuffix([rdy_datas], fileName2, null, "utf-8", false, "","|+|");
|
||||
XmBankFtpUtils.ftpUpload(fileName2, curDate)
|
||||
}
|
||||
}
|
||||
|
||||
+4
-4
@@ -150,8 +150,8 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def dist = dists[i]
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
|
||||
@@ -171,8 +171,8 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
// }
|
||||
// });
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
|
||||
+1
-1
@@ -62,7 +62,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
private static def districtMap = [:];
|
||||
private static def limit = 5000;
|
||||
|
||||
def static final SHORT_DATE = Integer.parseInt(com.fangjia.script.util.DateUtils.addMonth(com.fangjia.script.util.DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
|
||||
def static final SHORT_DATE = Integer.parseInt(com.fangjia.script.util.DateUtils.addMonth(com.fangjia.script.util.DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6, "无名称":7]
|
||||
|
||||
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋","幸福里","中国房价行情","常州住房"]
|
||||
|
||||
+4
@@ -76,6 +76,10 @@ class SynchDistrictPrice extends BaseBatch{
|
||||
data.put("build_type",d.build_type)
|
||||
data.put("c_date", new Date())
|
||||
data.put("m_date", new Date())
|
||||
// if(d.location){
|
||||
// data.put("lng", d.location.x)
|
||||
// data.put("lat", d.location.y)
|
||||
// }
|
||||
rets << data
|
||||
}
|
||||
})
|
||||
|
||||
+132
-81
@@ -1,16 +1,14 @@
|
||||
package com.fangjia.script.trade
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
|
||||
///导入价格指数
|
||||
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
@@ -19,10 +17,8 @@ import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.commons.lang.math.RandomUtils
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
@@ -53,7 +49,7 @@ class ImportIndexStats extends BaseBatch{
|
||||
|
||||
|
||||
|
||||
public static final def dingjiYear = 2020;
|
||||
public static def dingjiYear = 2020;
|
||||
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
private static final EsDataSource dataSource = new EsDataSource()
|
||||
@@ -74,24 +70,31 @@ class ImportIndexStats extends BaseBatch{
|
||||
|
||||
// updateDataType()
|
||||
// synch()
|
||||
// synchTjjIndex()//同步统计局抓取过来的指数
|
||||
// synchTjjIndex()//批量同步统计局抓取过来的指数
|
||||
|
||||
// Calendar calendar=Calendar.getInstance();
|
||||
// def month= calendar.get(Calendar.MONTH);
|
||||
// def year= calendar.get(Calendar.YEAR);
|
||||
// def date = createDateString(year, month, 0, "");
|
||||
// calculatePriceIndex(year, month, date);
|
||||
Calendar calendar=Calendar.getInstance();
|
||||
def month= calendar.get(Calendar.MONTH);
|
||||
def year= calendar.get(Calendar.YEAR);
|
||||
def date = createDateString(year, month, 0, "");
|
||||
|
||||
def day = calendar.get(Calendar.DATE)
|
||||
if(day == 5){
|
||||
calculatePriceIndex(year, month, date);
|
||||
}
|
||||
|
||||
createTjjData(date)//同步单期统计局数据
|
||||
|
||||
// addLostIndex()
|
||||
|
||||
addHistoryIndex()
|
||||
// addHistoryIndex()
|
||||
}
|
||||
|
||||
def addLostIndex(){
|
||||
def citys = CommonDatautils.findAllCity();
|
||||
def index = 0
|
||||
for(def city : citys){
|
||||
index++
|
||||
if(cityList.contains(city)){
|
||||
println city +" 数据来源于统计局"
|
||||
println "[${index}] "+ city +" 数据来源于统计局"
|
||||
continue;
|
||||
}
|
||||
def start = findStartOrEndIndex(city, 1)
|
||||
@@ -117,21 +120,48 @@ class ImportIndexStats extends BaseBatch{
|
||||
if(nextObj && nextObj.chain){
|
||||
indexs << nextObj.chain
|
||||
}
|
||||
udata.put("chain",DataHandleUtils.getDoubleRound(1, indexs.sum()/indexs.size()))
|
||||
udata.put("remark","推导")
|
||||
if(indexs){
|
||||
udata.put("chain",DataHandleUtils.getDoubleRound(1, indexs.sum()/indexs.size()))
|
||||
udata.put("remark","推导")
|
||||
}
|
||||
}
|
||||
if(!dbo.yeartoyear){
|
||||
def indexs = []
|
||||
if(lastMonthObj && lastMonthObj.yeartoyear){
|
||||
indexs << lastMonthObj.yeartoyear
|
||||
}
|
||||
if(nextObj && nextObj.yeartoyear){
|
||||
indexs << nextObj.yeartoyear
|
||||
}
|
||||
if(indexs){
|
||||
udata.put("yeartoyear",DataHandleUtils.getDoubleRound(1, indexs.sum()/indexs.size()))
|
||||
udata.put("remark","推导")
|
||||
}
|
||||
|
||||
}
|
||||
if(!dbo.fixedbase){
|
||||
def indexs = []
|
||||
if(lastMonthObj && lastMonthObj.fixedbase){
|
||||
indexs << lastMonthObj.fixedbase
|
||||
}
|
||||
if(nextObj && nextObj.fixedbase){
|
||||
indexs << nextObj.fixedbase
|
||||
}
|
||||
if(indexs){
|
||||
udata.put("fixedbase",DataHandleUtils.getDoubleRound(1, indexs.sum()/indexs.size()))
|
||||
udata.put("remark","推导")
|
||||
}
|
||||
|
||||
}
|
||||
if(udata){
|
||||
println city +" 补充"
|
||||
STATS_INDEX_COLL.update(new BasicDBObject("_id", dbo._id), new BasicDBObject('$set', udata),false, true);
|
||||
}
|
||||
// if(!dbo.yeartoyear){
|
||||
//
|
||||
// }
|
||||
// if(!dbo.fixedbase){
|
||||
//
|
||||
// }
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println "[${index}] "+ city +" 已处理完"
|
||||
}
|
||||
|
||||
}
|
||||
@@ -154,7 +184,7 @@ class ImportIndexStats extends BaseBatch{
|
||||
}
|
||||
|
||||
def addHistoryIndex(){
|
||||
def s = "202101"
|
||||
def s = "202001"
|
||||
def startDate = null;
|
||||
def stop = false;
|
||||
def runDates = []
|
||||
@@ -171,7 +201,7 @@ class ImportIndexStats extends BaseBatch{
|
||||
runDates << map
|
||||
s = DateUtils.addMonth(date, 1, "yyyyMM")
|
||||
|
||||
if(year==2024 && month==9){
|
||||
if(year==2024 && month==10){
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
@@ -182,6 +212,7 @@ class ImportIndexStats extends BaseBatch{
|
||||
|
||||
//计算统计局没有发布的城市的价格指数
|
||||
def calculatePriceIndex(def year, def month, def date){
|
||||
// def citys = ["定安","五指山","琼海","儋州","文昌","万宁","东方","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]//
|
||||
def citys = CommonDatautils.findAllCity();
|
||||
// def date = createDateString(year, month, 0, "");
|
||||
def haveIndexCitys = findPriceIndexCitys(date.toInteger());
|
||||
@@ -193,7 +224,7 @@ class ImportIndexStats extends BaseBatch{
|
||||
def newCitys = []
|
||||
for(def i=0;i<citys.size();i++) {
|
||||
def city = citys[i]
|
||||
if (!haveIndexCitys.contains(city)) {
|
||||
if (!haveIndexCitys.contains(city) && !cityList.contains(city)) {
|
||||
newCitys << city
|
||||
}
|
||||
}
|
||||
@@ -204,6 +235,15 @@ class ImportIndexStats extends BaseBatch{
|
||||
if(id){//已计算
|
||||
continue;
|
||||
}
|
||||
if(date.toInteger()>=203101){
|
||||
dingjiYear = 2030
|
||||
}else if(date.toInteger()>=202601){
|
||||
dingjiYear = 2025
|
||||
}else if(date.toInteger()>=202101){
|
||||
dingjiYear = 2020
|
||||
}else if(date.toInteger()>=201601){
|
||||
dingjiYear = 2015
|
||||
}
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
@@ -220,7 +260,8 @@ class ImportIndexStats extends BaseBatch{
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await(20, TimeUnit.SECONDS);
|
||||
latch.await(1, TimeUnit.MINUTES);
|
||||
// latch.await(1, TimeUnit.MINUTES);
|
||||
}
|
||||
|
||||
def calculating(city, year, month, date){
|
||||
@@ -647,6 +688,7 @@ class ImportIndexStats extends BaseBatch{
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("yearmonth", date);
|
||||
q.put("cat", "district")
|
||||
q.put("type", "城市")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
keys.put("city", 1)
|
||||
@@ -666,6 +708,7 @@ class ImportIndexStats extends BaseBatch{
|
||||
q.put("yearmonth", date);
|
||||
q.put("city", city);
|
||||
q.put("cat", "district");
|
||||
q.put("type", "城市")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
keys.put("city", 1)
|
||||
@@ -684,6 +727,7 @@ class ImportIndexStats extends BaseBatch{
|
||||
dbo.put("source",result.source);
|
||||
dbo.put("remark",result.remark);
|
||||
dbo.put("cat",result.cat);
|
||||
dbo.put("type",result.type);
|
||||
dbo.put("c_date",new Date());
|
||||
if(result.markup){
|
||||
dbo.put("chain", result.markup.toDouble());
|
||||
@@ -785,69 +829,76 @@ class ImportIndexStats extends BaseBatch{
|
||||
}
|
||||
|
||||
def createTjjData(date){
|
||||
def queryStartDate = DateUtils.parseFromTightByPattern(date+"01", "yyyyMMdd")
|
||||
def queryEndDate = DateUtils.parseFromTightByPattern(date+"05", "yyyyMMdd")
|
||||
def sellIndexMap = findTjjIndex(indices_price_house_sellColl, queryStartDate, queryEndDate)
|
||||
def sellIndexAreaMap = findTjjAreaIndex(indices_price_house_sell_typeColl, queryStartDate, queryEndDate)
|
||||
|
||||
def newIndexMap = findTjjIndex(indices_price_house_newColl, queryStartDate, queryEndDate)
|
||||
def newAreaIndexAreaMap = findTjjAreaIndex(indices_price_house_new_typeColl, queryStartDate, queryEndDate)
|
||||
def rets = []
|
||||
sellIndexMap.each{city, d->
|
||||
def dbo = new BasicDBObject()
|
||||
dbo.put("city",d.city)
|
||||
dbo.put("yearmonth",date.toInteger())
|
||||
dbo.put("source" ,1)
|
||||
dbo.put("remark" , "统计局数据导入")
|
||||
dbo.put("cat" , "district")
|
||||
dbo.put("c_date" , new Date())
|
||||
dbo.put("m_date" , new Date())
|
||||
dbo.put("type" , "城市")
|
||||
dbo.put("chain" ,d.mom)
|
||||
dbo.put("yeartoyear" , d.yoy)
|
||||
dbo.put("fixedbase" , d.fixedbase)
|
||||
def count = STATS_INDEX_COLL.count(new BasicDBObject("type","城市").append("source", 1).append("yearmonth", date.toInteger()))
|
||||
if(count == 0){
|
||||
def queryStartDate = DateUtils.parseFromTightByPattern(date+"01", "yyyyMMdd")
|
||||
def queryEndDate = DateUtils.parseFromTightByPattern(date+"05", "yyyyMMdd")
|
||||
def sellIndexMap = findTjjIndex(indices_price_house_sellColl, queryStartDate, queryEndDate)
|
||||
def sellIndexAreaMap = findTjjAreaIndex(indices_price_house_sell_typeColl, queryStartDate, queryEndDate)
|
||||
|
||||
def newIndexMap = findTjjIndex(indices_price_house_newColl, queryStartDate, queryEndDate)
|
||||
def newAreaIndexAreaMap = findTjjAreaIndex(indices_price_house_new_typeColl, queryStartDate, queryEndDate)
|
||||
def rets = []
|
||||
sellIndexMap.each{city, d->
|
||||
def dbo = new BasicDBObject()
|
||||
dbo.put("city",d.city)
|
||||
dbo.put("yearmonth",date.toInteger())
|
||||
dbo.put("source" ,1)
|
||||
dbo.put("remark" , "统计局数据导入")
|
||||
dbo.put("cat" , "district")
|
||||
dbo.put("c_date" , new Date())
|
||||
dbo.put("m_date" , new Date())
|
||||
dbo.put("type" , "城市")
|
||||
dbo.put("chain" ,d.mom)
|
||||
dbo.put("yeartoyear" , d.yoy)
|
||||
dbo.put("fixedbase" , d.fixedbase)
|
||||
|
||||
|
||||
def areaMap = sellIndexAreaMap.get(d.city)
|
||||
def areaObj1 = areaMap.get("90")
|
||||
dbo.put("below_chain" , areaObj1.mom)
|
||||
dbo.put("below_yeartoyear" , areaObj1.yoy)
|
||||
dbo.put("below_fixedbase" , areaObj1.fixedbase)
|
||||
def areaMap = sellIndexAreaMap.get(d.city)
|
||||
def areaObj1 = areaMap.get("90")
|
||||
dbo.put("below_chain" , areaObj1.mom)
|
||||
dbo.put("below_yeartoyear" , areaObj1.yoy)
|
||||
dbo.put("below_fixedbase" , areaObj1.fixedbase)
|
||||
|
||||
def areaObj2 = areaMap.get("90-144")
|
||||
dbo.put("among_chain" , areaObj2.mom)
|
||||
dbo.put("among_yeartoyear" , areaObj2.yoy)
|
||||
dbo.put("among_fixedbase" , areaObj2.fixedbase)
|
||||
def areaObj2 = areaMap.get("90-144")
|
||||
dbo.put("among_chain" , areaObj2.mom)
|
||||
dbo.put("among_yeartoyear" , areaObj2.yoy)
|
||||
dbo.put("among_fixedbase" , areaObj2.fixedbase)
|
||||
|
||||
def areaObj3 = areaMap.get("144")
|
||||
dbo.put("above_chain" , areaObj3.mom)
|
||||
dbo.put("above_yeartoyear" , areaObj3.yoy)
|
||||
dbo.put("above_fixedbase" , areaObj3.fixedbase)
|
||||
def areaObj3 = areaMap.get("144")
|
||||
dbo.put("above_chain" , areaObj3.mom)
|
||||
dbo.put("above_yeartoyear" , areaObj3.yoy)
|
||||
dbo.put("above_fixedbase" , areaObj3.fixedbase)
|
||||
|
||||
def newObj = newIndexMap.get(d.city)
|
||||
dbo.put("new_chain" , newObj.mom)
|
||||
dbo.put("new_yeartoyear" , newObj.yoy)
|
||||
dbo.put("new_fixedbase" , newObj.fixedbase)
|
||||
def newObj = newIndexMap.get(d.city)
|
||||
dbo.put("new_chain" , newObj.mom)
|
||||
dbo.put("new_yeartoyear" , newObj.yoy)
|
||||
dbo.put("new_fixedbase" , newObj.fixedbase)
|
||||
|
||||
def newAreaMap = newAreaIndexAreaMap.get(d.city)
|
||||
def areaNewObj1 = newAreaMap.get("90")
|
||||
dbo.put("new_below_chain" , areaNewObj1.mom)
|
||||
dbo.put("new_below_yeartoyear" , areaNewObj1.yoy)
|
||||
dbo.put("new_below_fixedbase" , areaNewObj1.fixedbase)
|
||||
def newAreaMap = newAreaIndexAreaMap.get(d.city)
|
||||
def areaNewObj1 = newAreaMap.get("90")
|
||||
dbo.put("new_below_chain" , areaNewObj1.mom)
|
||||
dbo.put("new_below_yeartoyear" , areaNewObj1.yoy)
|
||||
dbo.put("new_below_fixedbase" , areaNewObj1.fixedbase)
|
||||
|
||||
def areaNewObj2 = newAreaMap.get("90-144")
|
||||
dbo.put("new_among_chain" , areaNewObj2.mom)
|
||||
dbo.put("new_among_yeartoyear" , areaNewObj2.yoy)
|
||||
dbo.put("new_among_fixedbase" , areaNewObj2.fixedbase)
|
||||
def areaNewObj2 = newAreaMap.get("90-144")
|
||||
dbo.put("new_among_chain" , areaNewObj2.mom)
|
||||
dbo.put("new_among_yeartoyear" , areaNewObj2.yoy)
|
||||
dbo.put("new_among_fixedbase" , areaNewObj2.fixedbase)
|
||||
|
||||
def areaNewObj3 = newAreaMap.get("144")
|
||||
dbo.put("new_above_chain" , areaNewObj3.mom)
|
||||
dbo.put("new_above_yeartoyear" , areaNewObj3.yoy)
|
||||
dbo.put("new_above_fixedbase" , areaNewObj3.fixedbase)
|
||||
def areaNewObj3 = newAreaMap.get("144")
|
||||
dbo.put("new_above_chain" , areaNewObj3.mom)
|
||||
dbo.put("new_above_yeartoyear" , areaNewObj3.yoy)
|
||||
dbo.put("new_above_fixedbase" , areaNewObj3.fixedbase)
|
||||
|
||||
rets << dbo
|
||||
}
|
||||
if(rets){
|
||||
STATS_INDEX_COLL.insert(rets)
|
||||
}
|
||||
|
||||
rets << dbo
|
||||
}
|
||||
STATS_INDEX_COLL.insert(rets)
|
||||
}
|
||||
|
||||
def findTjjIndex(coll, queryStartDate, queryEndDate){
|
||||
|
||||
@@ -414,7 +414,7 @@ public class ExcelUtils{
|
||||
|
||||
def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
|
||||
def body = data.subList(1, data.size())
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 4
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 6
|
||||
println nThread
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
|
||||
final AtomicInteger counter = new AtomicInteger(0)
|
||||
|
||||
Reference in New Issue
Block a user