Merge remote-tracking branch 'origin/master'

This commit is contained in:
2024-11-07 14:53:02 +08:00
106 changed files with 8738 additions and 3742 deletions
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
/**
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年7月深圳二手房成交.xlsx", 0, 1)
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年9月深圳二手房成交.xlsx", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -1,11 +1,18 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 将国信达及成交对标数据加入到第三方均价表,做为均价参考
*/
@@ -13,19 +20,30 @@ class AddThirdPriceSample {
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
public static final DBCollection NO_PRICE_DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "deal_price_all", "mongo.db.fangjia.com")
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
static main(args) {
def task = new AddThirdPriceSample()
task.run()
// task.run() //添加第三方 fccs->保存的是国信达的参考价格 csjj->保存的是国信达的成交数据
task.addTrade() //添加成交
// task.matchOldData()
// task.noInCcbCity()
}
def run(){
for(def i=1;i<19;i++){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\级联待优化小区集合_7月.xlsx",{d->
if(d['A'] && d['C']){
for(def i=0;i<1;i++){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\国信达指导价录入-1024.xlsx",{d->
if(d['A'] && d['B']){
def dist = CommonDatautils.findDistrictById(d['A'].toString().trim())
if(dist){
def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202407)
def udata = new BasicDBObject("csjj_esf_price", d['C'].toString().toDouble().toInteger())
def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202409)
// def udata = new BasicDBObject("csjj_esf_price", d['C'].toString().toDouble().toInteger())
def udata = new BasicDBObject("fccs_esf_price", d['B'].toString().toDouble().toInteger())
AVERAGE_STATISTICS.update(q,new BasicDBObject('$set', udata),false, false)
}else{
println d['A'] +" 没查到"
@@ -37,4 +55,142 @@ class AddThirdPriceSample {
}
}
def addTrade(){
// def citys = ["武汉","南京","杭州","长沙","郑州","合肥","宁波","南通","嘉兴","珠海","泉州","绍兴","湖州","镇江"]
def rets = findTrades()
DEAL_PRICE_COLL.insert(rets)
}
def findTrades(){
def q = new BasicDBObject()
q.put("ccb_price_remark","建行无此城市成交")
// q.put("city", new BasicDBObject('$in', citys))
// q.put("flag", 1)
def dbcursor = NO_PRICE_DEAL_PRICE_COLL.find(q)
def rets = []
def index = 0
for(def db : dbcursor){
index++
def udata = new BasicDBObject()
udata.put("_id",db._id)
udata.put("url",db.url)
udata.put("area",db.area.toString().toDouble())
udata.put("avg_price",db.avg_price.toString().toDouble().toInteger())
udata.put("build_structure",db.build_structure)
udata.put("build_type",db.build_type)
udata.put("build_year",db.build_year)
udata.put("city",db.city)
udata.put("direction", db.direction)
udata.put("district_name",db.district_name)
udata.put("elevator",db.elevator)
udata.put("fitment",db.fitment)
udata.put("floor_info",db.floor_info)
if(db.hall){
udata.put("hall",db.hall.toString().toInteger())
}
if(db.height){
udata.put("height",db.height.toString().toInteger())
}
udata.put("housing_age",db.housing_age)
udata.put("lift",db.lift.toString().toInteger())
udata.put("lift_household",db.lift_household.toString().toInteger())
udata.put("property_ownership",db.property_ownership)
udata.put("region",db.region)
udata.put("room", db.room.toString().toInteger())
udata.put("room_structure",db.room_structure)
udata.put("source",db.source)
udata.put("total_price",db.total_price.toString().toDouble().toInteger())
udata.put("trade_date",db.trade_date)
udata.put("trade_property",db.trade_property)
udata.put("house_using",db.house_using)
udata.put("heating_type",db.heating_type)
udata.put("community_id",db.community_id)
udata.put("create_date",db.create_time)
udata.put("m_date", new Date())
udata.put("address",db.address)
udata.put("match_city",db.match_city)
udata.put("match_region",db.match_region)
udata.put("match_name",db.match_name)
udata.put("match_id",db.match_id)
udata.put("price_flag", "国信达成交-推导")
// DEAL_PRICE_COLL.save(udata)
println index
rets << udata
}
return rets
}
def matchOldData(){
def q = new BasicDBObject("source","贝壳找房").append("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2024-09-01","yyyy-MM-dd")))
def list = DEAL_PRICE_COLL.find(q).toList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(list.size())
for(def i=0;i<list.size();i++){
final def d = list[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def query = new BasicDBObject("url", d.url)
def obj = NO_PRICE_DEAL_PRICE_COLL.findOne(query)
if(obj){
def update = new BasicDBObject()
update.put("price_remark", "有价格")
NO_PRICE_DEAL_PRICE_COLL.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set', update), false,false)
}
println totalCount.get()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
def noInCcbCity(){
def citys = ["大理","苏州","无锡","通辽","绵阳","吉安","潍坊","襄阳","济宁","张家口","泸州","乐山","赣州","九江","眉山","桂林","雅安","广州","景德镇","上海","运城","内江","金华","濮阳","丹东","淄博","新乡","廊坊","烟台","邯郸","萍乡","宜宾","咸阳","许昌","德阳","银川","洛阳","天津","湛江","长沙","深圳","湘西","信阳","成都","威海","福州","上饶","常州","宜春","天水","平顶山","常德","吉林","盐城","杭州","赤峰","南京","周口","济南","株洲","大连","徐州","泰安","太原","凉山","兰州","承德","广元","台州","南昌","乌鲁木齐","海口","北京","淮安","清远","临沂","柳州","衢州","攀枝花","呼和浩特","昆明","岳阳","衡阳","温州","石家庄","连云港","保定","阜阳","遂宁","菏泽","青岛","唐山","开封","漳州","贵阳","驻马店","宝鸡","达州","汉中","南充","遵义","包头","南宁","沈阳","重庆","芜湖","厦门","大同"]
def q = new BasicDBObject()
q.put("price_remark",new BasicDBObject('$ne', "有价格"))
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2024-09-01","yyyy-MM-dd")))
q.put("avg_price",new BasicDBObject('$gt', 0))
q.put("match_id",new BasicDBObject('$exists', true))
def list = NO_PRICE_DEAL_PRICE_COLL.find(q).toList()
println "共:"+list.size()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(list.size())
for(def i=0;i<list.size();i++){
final def d = list[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
if(!citys.contains(d.city)){
def update = new BasicDBObject()
update.put("ccb_price_remark", "建行无此城市成交")
NO_PRICE_DEAL_PRICE_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set', update), false,false)
}
println totalCount.get()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
}
@@ -68,14 +68,14 @@ class BigdataDealBatch extends BaseBatch{
//664a22000000000000000000 5-20
//664b73800000000000000000 5-21
def citys =CcbCommonUtils.getRunningCitys(args)
def citys = CcbCommonUtils.getRunningCitys(args)
println "curDate:"+curDate
def allCitys = CommonDatautils.findAllCity()
for(def city : allCitys){
addDealPriceMatch(city);
}
updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
// def allCitys = CommonDatautils.findAllCity()
// for(def city : allCitys){
// addDealPriceMatch(city);
// }
// updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
def rows = []
@@ -720,38 +720,38 @@ class BigdataDealBatch extends BaseBatch{
if(!t.match_id){
def region = RegionData.formatRegion(t.city, t.region)
def rMap = null
if(region || (!region && ["澄迈"].contains(t.city))) {
if(t.house_use && t.house_use.contains("商住")){
if(!t.name.contains("商住")){
t.name = t.name +"商住楼"
}
if(t.house_use && t.house_use.contains("商住")){
if(!t.name.contains("商住")){
t.name = t.name +"商住楼"
}
def flag = false
def address = null
if(t.address){
flag = t.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷).+[弄号]$')
if(flag){
address = t.address
}
}
def flag = false
def address = null
if(t.address){
flag = t.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷).+[弄号]$')
if(flag){
address = t.address
}
}
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name, address);
if(!rMap && !t.name.toString().endsWith("小区")){
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name+"小区", address);
}
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name, address);
if(!rMap && !t.name.toString().endsWith("小区")){
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name+"小区", address);
}
if(rMap && rMap.region==region){
if(rMap && region && rMap.region==region){
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
}else if(rMap && rMap.address && address && region && rMap.region!=region && address.contains(rMap.address)){
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
}else if (rMap && ["澄迈"].contains(rMap.city)){
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
}else if(rMap && rMap.region!=t.region && ["开发区","八里湖新区","经济技术开发区(头屯河区)","角美台商投资区","贵州双龙航空港经济区","东津新区"].contains(t.region)){
def lists = CommonDatautils.findDistrictList(rMap.city, null, rMap.name, "district")
if(lists && lists.size()==1){
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
}else if(rMap && rMap.address && address && rMap.region!=region && address.contains(rMap.address)){
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
}else if (rMap && ["澄迈"].contains(rMap.city)){
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
}else{
if (rMap) {
println "区域不同"
}
}else {
println "多区域"
}
}
println count_.get()+" "+t+" "+ rMap
}
@@ -40,7 +40,7 @@ class CcbCommonUtils {
public static final DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com")
// public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "deal_price_all", "mongo.db.fangjia.com")
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017)
@@ -237,7 +237,7 @@ class CcbCommonUtils {
def lists = []
println dist.name +" "+total
for(def d : dbcursor){
def key=d.city+"##"+d.match_region+"##"+d.match_name+"##"+d.area+"##"+d.avg_price+"##"+d.total_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
def key=d.city+"##"+d.match_region+"##"+d.match_name+"##"+d.area+"##"+d.avg_price+"##"+d.total_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")+"##"+d.source
if(!rets.contains(key)){
rets << key
d.dist_id = dist._id.toString()
@@ -660,6 +660,7 @@ class CcbCommonUtils {
udata.put("match_region", match_region)
udata.put("match_name", match_name)
udata.put("match_id", match_id)
udata.put("m_date", new Date())
DEAL_PRICE_COLL.update(new BasicDBObject("_id",id), new BasicDBObject('$set',udata))
}
@@ -735,6 +736,7 @@ class CcbCommonUtils {
def q = new BasicDBObject()
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
q.put("_id", new BasicDBObject('$gte', cDate))
q.put("price_flag", new BasicDBObject('$exists', false))
def count = DEAL_PRICE_COLL.count(q)
if(count>0){
def udata = new BasicDBObject()
@@ -5,9 +5,14 @@ import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import groovy.sql.Sql
import org.bson.types.ObjectId
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 建行成交数据表优化
*/
@@ -24,6 +29,8 @@ class DealPriceDataProcess {
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
static main(args) {
def task = new DealPriceDataProcess()
// task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据
@@ -394,43 +401,73 @@ class DealPriceDataProcess {
def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def set = new HashSet()
def index = 0
for(def city : citys) {
index ++
if(index<145){
continue
}
println index +" "+city
def cityid = cityCodeDatas.get(city)
def rets = findPeGteThirdParty2(cityid, DateUtils.formatDateByPattern(new Date(),"yyyyMM"))//当前月日期
println city +" "+rets.size()
if(rets){
set.add(city)
}
for(def m : rets){
// def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
// sqlInstance236.executeUpdate(sql)
def querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
def total = 0
pgsqlInstance.eachRow(querySql, { r ->
total = r.count
})
if(total>0){
println city +" "+m.commid+" (按价格)删除:"+total
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
pgsqlInstance.executeUpdate(sql)
}else{
querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} "
pgsqlInstance.eachRow(querySql, { r ->
total = r.count
})
if(total>0){
println city +" "+m.commid+" (按小区)删除:"+total
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} "
pgsqlInstance.executeUpdate(sql)
}
}
}
processDetele(city, cityid, month, rets)
}
println set.toString()
}
def processDetele(city, cityid, month, rets){
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
final def m = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
// def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
// sqlInstance236.executeUpdate(sql)
def querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
def total = 0
pgsqlInstance.eachRow(querySql, { r ->
total = r.count
})
if(total>0){
println city +" "+m.commid+" (按价格)删除:"+total
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
pgsqlInstance.executeUpdate(sql)
}else{
querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} "
pgsqlInstance.eachRow(querySql, { r ->
total = r.count
})
if(total>0){
println city +" "+m.commid+" (按小区)删除:"+total
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} "
pgsqlInstance.executeUpdate(sql)
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
def findThirdCount(sqlInstance236, sql){
def count = 0
sqlInstance236.eachRow(sql,{r->
@@ -631,36 +631,36 @@ def findAllData(sqlInstance236, citycode){
//},0)
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\添加别名.xlsx",{d->
if(d['A'] && d['B']){
def dbo = CommonDatautils.findDistrictById(d['A'])
if(dbo){
def alias = null
if(!dbo.alias){
alias = []
}else{
alias = dbo.alias
}
if(!alias.contains(d['B'])){
alias << d['B']
}
if(alias){
def udata = new BasicDBObject()
udata.put("alias", alias)
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", dbo._id)
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}else{
println d['A']+" "+"别名已存在"
}
}else{
println dbo._id.toString() +" 没找到"
}
}
})
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\添加别名.xlsx",{d->
// if(d['A'] && d['B']){
// def dbo = CommonDatautils.findDistrictById(d['A'])
// if(dbo){
// def alias = null
// if(!dbo.alias){
// alias = []
// }else{
// alias = dbo.alias
// }
// if(!alias.contains(d['B'])){
// alias << d['B']
// }
// if(alias){
// def udata = new BasicDBObject()
// udata.put("alias", alias)
// udata.put("m_date",new Date())
//
// def q = new BasicDBObject()
// q.put("_id", dbo._id)
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
// }else{
// println d['A']+" "+"别名已存在"
// }
//
// }else{
// println dbo._id.toString() +" 没找到"
// }
// }
//})
//def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]
//for(def c : citys){
@@ -1234,15 +1234,7 @@ def findThirdCommCount(cityid, sqlInstance){
//}
//
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\无价格小区(3)2.23.xlsx"
//ExcelUtils.multiThreadingModify(fileName, { d ->
// def dist = CommonDatautils.findDistrictById(d['A'])
// if(dist){
// d['G'] = dist.city
// d['H'] = dist.region
// d['I'] = dist.name
// }
//})
@@ -1331,10 +1323,10 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240806--国信达价格录入.xlsx"
//
//def fileName = "D:\\数据\\建行\\房产估值\\6个城市待优化_9月xlsx.xlsx"
////
//ExcelUtils.multiThreadingModify(fileName, {d->
// if(d['J']){
// if(d['G']){
//// def region = RegionData.formatRegion(d['A'], d['J'])
//// if(region){
//// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district")
@@ -1344,23 +1336,34 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
//// d['L'] = dist.status-dist.name_cat
//// }
//// }
//// if(d['J'] instanceof String){
// def dist = CommonDatautils.findDistrictById(d['J'])
// if(d['G'] instanceof String){
// def dist = CommonDatautils.findDistrictById(d['G'])
// if(dist){
// d['A'] = dist.city
// d['B'] = dist.region
// d['C'] = dist.name
// d['K'] = dist.estate_type2
// d['L'] = dist.status
// d['M'] = dist.name_cat
//// d['A'] = dist.city
//// d['B'] = dist.region
//// d['C'] = dist.name
//// d['K'] = dist.estate_type2
//// d['L'] = dist.status
//// d['M'] = dist.name_cat
//
//// d['H'] = dist.name
// d['H'] = dist.name
// }
//// }
// }
// }
//},0)
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20241025人工指导价.xlsx"
ExcelUtils.multiThreadingModify(fileName, { d ->
def dist = CommonDatautils.findDistrictById(d['J'])
if(dist){
d['A'] = dist.city
d['B'] = dist.region
d['C'] = dist.name
}
})
//def file = "D:\\数据\\建行\\房产估值\\指导价\\5城价格翻新6-21.xlsx"
//ExcelUtils.multiThreadingModify(file, { d ->
// if(d['A']){
@@ -47,7 +47,8 @@ class TradeStatLog {
// task.run()//统计最近3个月抓取的成交数据量
// task.exportTrade()//导出未入库成交
task.exportTrade2()//导出未入库成交
// task.exportTrade2()//导出未入库成交
task.updateDealPriceRegion()//修改成交区域数据
// task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
// task.averageStat()
@@ -597,7 +598,7 @@ class TradeStatLog {
// def endDate = "2024-07-01"
def ids = findAllDealIds();
println ids.size()
ExcelUtils.multiThreadingModify("D:\\数据\\其它\\2024年6月成交数据.xlsx",{d->
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\建行成交\\廊坊.xlsx",{d->
if(ids.contains(d['A'])){
d['V'] = "供"
}
@@ -674,7 +675,7 @@ class TradeStatLog {
}
def findAllDealIds(){
def sql = "select gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where p9_start_date='2024-07-28'"
def sql = "select gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where urbn_nm='廊坊'"
def ids = []
pssqlInstance.eachRow(sql,{r->
if(!ids.contains(r.gujia_bds_mdl_id)){
@@ -801,6 +802,20 @@ class TradeStatLog {
},0)
}
def updateDealPriceRegion(){
ExcelUtils.modify("D:\\数据\\建行\\建行成交\\成交待修改区域数据.xlsx",{d->
if(d['A'] && d['G']){
def region = RegionData.formatRegion(d['D'], d['G'])
if(region){
def udata = new BasicDBObject()
udata.put("region", region)
DEAL_PRICE_COLL.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata))
}
}
},0)
}
def findTradeById(id){
def q = new BasicDBObject()
@@ -24,6 +24,13 @@ class AddressCut extends BaseBatch{
final DBCollection thirdPartyPriceColl = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018)
final DBCollection thirdPartyPriceBkColl = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price_history_bk_new", "192.168.0.235", 27018)
final DBCollection dataProcessColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process", "192.168.0.235", 27018)
final DBCollection dataProcessBkColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process_history_bk", "192.168.0.235", 27018)
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
static {
@@ -39,50 +46,66 @@ class AddressCut extends BaseBatch{
@Override
public void process(Object args) {
<<<<<<< HEAD
// sameAddr()
seaweedStreetUpdate()
def dates = [202408]
for(def date : dates){
// synch(date)
synchDataProcess(date)
println date + "同步完成! "
}
}
def seaweedStreetUpdate(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区地址拆分\\已核实-路2.xlsx",{d->
if(d['A']){
def dbo = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
if(dbo){
def udate = new BasicDBObject()
def street = null
def address = ""
if(d['E']){
street = d['E']
udate.put("street", street)
address = street
}
def streetNum = null
if(d['F']){
streetNum = d['F']
udate.put("street_number", streetNum)
address = address + streetNum
}
if(dbo.address!=address){
udate.put("address", address)
}
if(udate){
udate.put("m_date", new Date())
DBUtils.seaweed.update(new BasicDBObject('_id', dbo.get("_id")), new BasicDBObject('$set', udate))
}
}
def synch(date){
def q = new BasicDBObject()
q.put("s_date", date)
def dbcursor = thirdPartyPriceColl.find(q).batchSize(5000)
def index = 0
def rets = []
def num = 0
for(def d : dbcursor){
num++
index++
rets << d
if(index%10000==0){
thirdPartyPriceBkColl.insert(rets)
println(num)
rets.clear()
}
})
=======
// sameAddr() //相同地址小区
// roadMatch()
addAnjukeRoad()
}
if(rets.size()>0){
thirdPartyPriceBkColl.insert(rets)
rets.clear()
}
}
def synchDataProcess(date){
def q = new BasicDBObject()
q.put("s_date", date)
def dbcursor = dataProcessColl.find(q).batchSize(5000)
def index = 0
def rets = []
def num = 0
for(def d : dbcursor){
num++
index++
rets << d
if(index%10000==0){
dataProcessBkColl.insert(rets)
println(num)
rets.clear()
}
}
if(rets.size()>0){
dataProcessBkColl.insert(rets)
rets.clear()
}
}
def addAnjukeRoad(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区地址拆分\\小区地址切分(结果)_1.xlsx", { d ->
def roads = findThirdPartyPrice(d['A'], d['B'], d['C'], "链家")
@@ -130,8 +153,6 @@ class AddressCut extends BaseBatch{
}
})
>>>>>>> c1a5ab8c6aa973a91e6be5547ec0925db3fa500f
}
def sameAddr() {
@@ -1457,45 +1457,45 @@ class ExportDistrictRongHe extends BaseBatch{
println "community_p_describe_gj 上传Ftp完成......"
}
def exportGuideStatistics(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
}
def headers = []
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = null;
if(CcbCommonUtils.citys_92.contains(city)){
sql = "select MODEL_TYPE,CITYID,CITYNAME,YEAR_MONTH,COMM_SUM_COUNT,COMM_COUNT,RATIO,TYPE from GUIDE_STATISTICS where cityid='${cityCode}' and year_month=${time}";
}else{
sql = "select MODEL_TYPE,CITYID,CITYNAME,YEAR_MONTH,COMM_SUM_COUNT,COMM_COUNT,RATIO,TYPE from GUIDE_STATISTICS where cityid='${cityCode}'";
}
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.MODEL_TYPE
data << r.CITYID
data << r.CITYNAME
data << r.YEAR_MONTH
data << r.COMM_SUM_COUNT
data << r.COMM_COUNT
data << r.RATIO
data << r.TYPE
allRows << data
});
}
println allRows.size()
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "guide_statistics 数据文件输出完成......"
CcbFtpUtils.ftpUpload("rh", fileName, curDate)
println "guide_statistics 上传Ftp完成......"
}
// def exportGuideStatistics(time){
//
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
// def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
// if(CommonDatautils.isWindow()){
// fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
// }
// def headers = []
// def allRows = []
//
// for(def city : citys) {
// def cityCode = cityCodeMap.get(city)
// def sql = null;
// if(CcbCommonUtils.citys_92.contains(city)){
// sql = "select MODEL_TYPE,CITYID,CITYNAME,YEAR_MONTH,COMM_SUM_COUNT,COMM_COUNT,RATIO,TYPE from GUIDE_STATISTICS where cityid='${cityCode}' and year_month=${time}";
// }else{
// sql = "select MODEL_TYPE,CITYID,CITYNAME,YEAR_MONTH,COMM_SUM_COUNT,COMM_COUNT,RATIO,TYPE from GUIDE_STATISTICS where cityid='${cityCode}'";
// }
//
// sqlInstance.eachRow(sql, { r ->
// def data = []
// data << r.MODEL_TYPE
// data << r.CITYID
// data << r.CITYNAME
// data << r.YEAR_MONTH
// data << r.COMM_SUM_COUNT
// data << r.COMM_COUNT
// data << r.RATIO
// data << r.TYPE
// allRows << data
// });
// }
// println allRows.size()
// FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
//
// println "guide_statistics 数据文件输出完成......"
// CcbFtpUtils.ftpUpload("rh", fileName, curDate)
// println "guide_statistics 上传Ftp完成......"
// }
def exportPgGuideStatistics(time){
@@ -1514,7 +1514,6 @@ class ExportDistrictRongHe extends BaseBatch{
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_model_type
data << r.urbn_id
data << r.urbn_nm
data << r.gujia_clc_yrmo
@@ -1522,6 +1521,7 @@ class ExportDistrictRongHe extends BaseBatch{
data << r.gujia_comm_count
data << r.gujia_ratio
data << r.gujia_type
data << r.gujia_model_type
allRows << data
});
}
@@ -1,43 +1,22 @@
package com.fangjia.script.commData
import com.fangjia.core.RegionData
import com.fangjia.datastore.driver.FacetResult
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.JacksonUtil
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.AggregationOutput
import com.mongodb.client.MongoCursor
import com.mongodb.client.model.Accumulators
import com.mongodb.client.model.Aggregates
import com.rabbitmq.client.impl.AMQImpl
import groovy.json.JsonSlurper
import net.sf.json.JSONArray
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.apache.poi.ss.util.RegionUtil
import org.bson.Document
import org.bson.types.ObjectId
import com.fangjia.core.util.HttpRequest;
import com.fangjia.core.util.HttpRequest
import com.fangjia.datastore.driver.DSCollection
import com.fangjia.datastore.driver.DSCursor
import com.fangjia.datastore.driver.DataStore
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.newDistrict.InvokeHttpRequest
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.GeoUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBCursor
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import com.fangjia.script.util.*
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.*
import groovy.json.JsonSlurper
import net.sf.json.JSONArray
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
@@ -122,6 +101,8 @@ class CommonDatautils {
public static final DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
public static final DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com")
public static final DBCollection STATS_INDEX_COLL = MongoUtils.getCollectionByName("fangjia_stat", "stats_index", "mongo.db.fangjia.com")
public static DSCollection getHouseDataStoreConnection(){
if(house_dataStoreConnection != null) {
return house_dataStoreConnection
@@ -820,6 +801,7 @@ class CommonDatautils {
DBObject q = new BasicDBObject();
q.put("status",0)
// q.put("status",new BasicDBObject('$ne', 0))
q.put("cat", cat);
q.put("city", city)
@@ -829,7 +811,7 @@ class CommonDatautils {
keys.put(field, 1);
}
}
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(2000)
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(5000)
def rets = []
for(def d : dbsour){
rets << d
@@ -959,13 +941,7 @@ class CommonDatautils {
keys.put(field, 1);
}
}
def cursor = getDataStoreConnection().find(q, keys).batchSize(2000)
if(cursor){
for(def obj : cursor){
return obj;
}
}
return null;
return SEAWEED_COLL.findOne(q, keys)
}
@@ -1132,6 +1108,10 @@ class CommonDatautils {
q.put("city", city);
q.put("cat", cat);
q.put("name", name);
if(region){
q.put("region", region);
}
q.put("status",new BasicDBObject('$ne', 0))
DBObject keys = new BasicDBObject();
if(fields){
@@ -1680,7 +1660,7 @@ class CommonDatautils {
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
queryCondition.put("category","region");
queryCondition.put("show",1);
// queryCondition.put("show",1);
def dbCursor = REGION_BLOCK_COLL.find(queryCondition)
def regions = [];
for(DBObject obj : dbCursor){
@@ -1892,6 +1872,33 @@ class CommonDatautils {
}
public static def queryFirstOrLastDistrictPrice(city, region, name, sort){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
queryCondition.put("region", region);
queryCondition.put("district_name", name);
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject("weekend", sort)).limit(1)
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def dbo = null;
while(cur.hasNext()) {
dbo = cur.next()
}
return dbo;
}
public static def queryDistrictPriceByWeekend(city, region, name, weekend){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
queryCondition.put("region", region);
queryCondition.put("district_name", name);
queryCondition.put("weekend", weekend);
return DIST_STAT_COLL.findOne(queryCondition)
}
public static def queryDistrictPrice(city, region, name, weekend, cat){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
@@ -2351,6 +2358,25 @@ class CommonDatautils {
}
return price;
}
public static def findCityOrRegionPrice(city, region, weekend){
DBObject q = new BasicDBObject()
q.put("city", city);
if(region){
q.put("region", region);
q.put("cat", "区域")
}else{
q.put("cat", "城市")
}
q.put('weekend', weekend);
def dbCursor = REGION_STAT_COLL.find(q).sort(new BasicDBObject().append("weekend", -1)).limit(1);
def price = null;
for(def obj : dbCursor){
price = obj.get("avg_price");
}
return price;
}
public static def findDistrictPrice(def city, def region, def name){
DBObject queryCondition = new BasicDBObject()
@@ -3414,6 +3440,54 @@ class CommonDatautils {
return price;
}
static def queryCityOrRegionPrice(city, region, sort){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
if(region){
queryCondition.put("region", region);
queryCondition.put("cat", "区域")
}else{
queryCondition.put("cat", "城市")
}
DBCursor cur = REGION_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", sort)).limit(1)
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def obj = 0;
while(cur.hasNext()) {
obj = cur.next()
}
return obj;
}
static def queryFirstOrLastRentPrice(city, region, name, sort){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
if(region){
queryCondition.put("region", region);
}
if(name){
queryCondition.put("cat", "district");
queryCondition.put("district_name", name)
}else{
if(region){
queryCondition.put("cat", "region")
}else{
queryCondition.put("cat", "city")
}
}
DBCursor cur = SQMETRE_RENT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", sort)).limit(1)
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def obj = 0;
while(cur.hasNext()) {
obj = cur.next()
}
return obj;
}
static def queryCityRegionPrice(city, region, startDate, endDate){
DBObject queryCondition = new BasicDBObject()
@@ -3451,6 +3525,20 @@ class CommonDatautils {
return prices;
}
static def queryCityPriceByWeekend(city, weekend){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
queryCondition.put("cat", "城市");
queryCondition.put('weekend', DateUtils.parseFromTightYYYYMMDD(weekend));
def obj = REGION_STAT_COLL.findOne(queryCondition)
def prices = null;
if(obj) {
prices = obj.get("avg_price");
}
return prices;
}
@@ -3630,11 +3718,26 @@ class CommonDatautils {
return count;
}
static def findSqmetreRent(def city, def region, def name, def weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("district_name", name)
if(region){
q.put("region", region)
}
if(name){
q.put("district_name", name)
q.put("cat", "district")
}else{
if(!region){
if(city){
q.put("cat", "city")
}
}else{
q.put("cat", "region")
}
}
q.put("weekend", weekend)
def dbcur = SQMETRE_RENT_COLL.find(q)
def dbo = null
@@ -3989,6 +4092,21 @@ class CommonDatautils {
return retMap;
}
static def findHouseCount(def city, def region, def name){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
.must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("district_name", name)).must(QueryBuilders.termQuery("cat", "sell"))
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
def index = new Random().nextInt(5);
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
return houses.size();
}
/**
* 是否是别墅
* @return
@@ -4660,7 +4778,6 @@ class CommonDatautils {
def q = new BasicDBObject()
q.put("type","区域")
q.put("district_diff",1)
q.put("city",new BasicDBObject('$in', ["中山","东莞","东方"]))
def keys = new BasicDBObject()
keys.put("province",1)
@@ -4689,12 +4806,97 @@ class CommonDatautils {
return regionMap
}
static def queryAllRegionNameByCity(){
def q = new BasicDBObject()
q.put("type","区域")
q.put("district_diff",1)
def keys = new BasicDBObject()
keys.put("city_short",1)
keys.put("district_short",1)
keys.put("alias",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def retMap = [:]
for(def d : dbcursor){
def regionMap = retMap.get(d.city_short)
if(!regionMap){
regionMap = [:]
retMap.put(d.city_short, regionMap)
}
regionMap.put(d.district_short, d.alias)
}
return retMap
}
static def getRegionShortName(city){
def q = new BasicDBObject()
q.put("type","区域")
q.put("district_diff",1)
q.put("city", city)
def keys = new BasicDBObject()
keys.put("district",1)
keys.put("district_short", 1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def shortNameMap = [:]
for(def d : dbcursor){
shortNameMap.put(d.district_short, d.district)
}
return shortNameMap
}
static def queryAllRegionFullNameByCity(){
def q = new BasicDBObject()
q.put("type","区域")
q.put("district_diff",1)
def keys = new BasicDBObject()
keys.put("city",1)
keys.put("district",1)
keys.put("alias",1)
keys.put("mzb_district_name",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def retMap = [:]
for(def d : dbcursor){
def regionMap = retMap.get(d.city)
if(!regionMap){
regionMap = [:]
retMap.put(d.city, regionMap)
}
regionMap.put(d.district, d)
}
return retMap
}
static def queryAllStreetNameByCity(city){
def q = new BasicDBObject()
q.put("type","乡镇街道")
q.put("city", city)
def keys = new BasicDBObject()
keys.put("street",1)
keys.put("alias",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def retMap = [:]
for(def d : dbcursor){
retMap.put(d.street, d.alias)
}
return retMap
}
static def queryCommunityByCity(){
def q = new BasicDBObject()
q.put("type","乡镇街道")
// q.put("province",new BasicDBObject('$in', ['浙江省','江苏省','山东省']))
q.put("city",new BasicDBObject('$in', ["中山市","东莞市","东方市"]))
q.put("city",new BasicDBObject('$in', ["深圳市"]))
def keys = new BasicDBObject()
keys.put("province",1)
@@ -4719,6 +4921,75 @@ class CommonDatautils {
return townMap
}
static def queryMzbRegion(city){
def q = new BasicDBObject()
q.put("type","区域")
q.put("city",city)
def keys = new BasicDBObject()
keys.put("district",1)
keys.put("mzb_district_name",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def regionMap = [:]
for(def d : dbcursor){
regionMap.put(d.district, d.mzb_district_name)
}
return regionMap
}
static def queryTownByCity(city, fields, mzbRegionMap){
def q = new BasicDBObject()
q.put("type","乡镇街道")
q.put("city",city)
def keys = new BasicDBObject()
for(def key : fields){
keys.put(key,1)
}
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def townMap = [:]
for(def d : dbcursor){
def key = mzbRegionMap.get(d.district)
def list = townMap.get(key)
if(!list){
list = []
}
list << d
townMap.put(key, list)
}
println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss")
return townMap
}
static def queryTownByCity(city, fields){
def q = new BasicDBObject()
q.put("type","乡镇街道")
q.put("city",city)
def keys = new BasicDBObject()
for(def key : fields){
keys.put(key,1)
}
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def townMap = [:]
for(def d : dbcursor){
if(d.mzb_district_name){
def list = townMap.get(d.mzb_district_name)
if(!list){
list = []
}
list << d
townMap.put(d.mzb_district_name, list)
}
}
println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss")
return townMap
}
static def queryTownByCity(city){
@@ -4755,7 +5026,7 @@ class CommonDatautils {
def q = new BasicDBObject()
q.put("type","乡镇街道")
q.put("city",new BasicDBObject('$in', ["中山","东莞","东方"]))
def keys = new BasicDBObject()
keys.put("city",1)
@@ -4954,4 +5225,38 @@ class CommonDatautils {
}
static def findStatIndexBySort(type, city, region, sort){
def q = new BasicDBObject()
q.put("cat", "district")
q.put("type", type)
if(region){
q.put("district", region)
}
if(city){
q.put("city", city)
}
def cur = STATS_INDEX_COLL.find(q).sort(new BasicDBObject("yearmonth", sort)).limit(1)
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def obj = 0;
while(cur.hasNext()) {
obj = cur.next()
}
return obj;
}
static def findStatIndexByMonth(type, city, region, month){
def q = new BasicDBObject()
q.put("cat", "district")
q.put("type", type)
if(region){
q.put("district", region)
}
if(city){
q.put("city", city)
}
q.put("yearmonth", month)
return STATS_INDEX_COLL.findOne(q)
}
}
@@ -202,13 +202,18 @@ class DistrictSellStatBatch extends BaseBatch {
if(curPrice!=curPriceObj.avg_price){
def data = new BasicDBObject()
data.put("m_date", new Date())
data.put("commid", dist._id.toString())
// data.put("commid", dist._id.toString())
data.put("mom", mom)
data.put("yoy", yoy)
data.put("avg_price",curPrice)
data.put("value_type", weekendPriceObj.value_type)
data.put("credit", weekendPriceObj.credit)
DISTRICT_SELL_STAT.update(new BasicDBObject("_id", curPriceObj._id) , new BasicDBObject('$set', data),true,false)
def q = new BasicDBObject()
q.put("date", curMonth)
q.put("commid", dist._id.toString())
DISTRICT_SELL_STAT.update(q , new BasicDBObject('$set', data),true,false)
// DISTRICT_SELL_STAT.update(new BasicDBObject("_id", curPriceObj._id) , new BasicDBObject('$set', data),true,false)
println "[update]"+totalCount.get()+" "+ dist.city+" "+dist.name
}
}else{
@@ -2,8 +2,10 @@ package com.fangjia.script.gj
import com.fangjia.core.BlockData
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.DateUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
@@ -531,6 +533,28 @@ class FiledStrand {
return null
}
static def processLngAndLat(val) {
if (val) {
def map = [:]
def lng = DataHandleUtils.getDoubleRound(6, val.x)
def lat = DataHandleUtils.getDoubleRound(6, val.y)
Long gd_lat = lat * 10000000000L
Long gd_lng = lng * 10000000000L
map.lat = gd_lat
map.lng = gd_lng
def baidu = BaiDuApiUtils.geoconv(lng, lat, 3)
Long bd_lng = baidu.x * 10000000000L
Long bd_lat = baidu.y * 10000000000L
map.lat2 = bd_lat
map.lng2 = bd_lng
return map
}
return null
}
static def isRoad(def s){
def pattern = Pattern.compile("[·|½Ö|´óµÀ|Ïï]");
def matcher = pattern.matcher(s)
@@ -2,33 +2,97 @@ package com.fangjia.script.gj
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\成交未匹配.xlsx",{d->
if(d['A'] && !d['E']){
// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'].toString(), "district")
// def dist = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'].toString(), "district")
println d['A'] +" "+d['C']
// def total = findDatas(d['A'], d['B'], d['C'])
// d['D'] = total
def c = getAmapLocation(d['A']+"市", d['B']+"区", d['C'].replace("?","").replace("·","").replace("?","").replace("|","").replace("·",""))
d['E'] = c
}
})
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
//ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\成交未匹配.xlsx",{d->
// if(d['A'] && !d['E']){
//// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'].toString(), "district")
//// def dist = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'].toString(), "district")
// println d['A'] +" "+d['C']
//// def total = findDatas(d['A'], d['B'], d['C'])
//// d['D'] = total
// def c = getAmapLocation(d['A']+"市", d['B']+"区", d['C'].replace("?","").replace("·","").replace("?","").replace("|","").replace("·",""))
// d['E'] = c
// }
//})
def URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection seaweed_coll = MongoUtils.getCollectionByName2("fangjia", "seaweed", URL)
def q = new BasicDBObject()
q.put("city", "上海市")
q.put("category", "district")
q.put("status", 0)
def ret = []
def list = seaweed_coll.find(q).batchSize(5000).toList()
println "总计:"+list.size()
//for(def l : list){
// def obj = getAmapLocation(l.city, l.region, l.name)
// if(obj){
// ret << [l.city, l.region, l.name, obj.cityname, obj.adname, obj.name,obj.type]
// }
//}
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def addCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(list.size())
def total = list.size()
for(def i=0;i<list.size();i++){
final def l = list[i]
def dist = CommonDatautils.findDistrict(l.fangjia_city, l.fangjia_region, l.name, "district")
fixedThreadPool.execute(new Runnable() {
void run() {
try{
addCount.addAndGet(1)
def alias = []
alias << dist.name
if(l.alias){
alias.addAll(l.alias)
}
for(def alia : alias){
def obj = getAmapLocation(l.city, l.region, alia)
if(obj){
ret << [l.city, l.region, l.name, obj.cityname, obj.adname, obj.name,obj.type]
break
}
}
println addCount.get()+"/"+total+" "+ ret.size()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
ExcelUtils.write("D:\\数据\\数据建设\\小区\\高德地图有围栏数据.xlsx", ret, "高德")
def getAmapLocation(city, region, name){
String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') "
def total = 0
String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
def obj = null
ElasticUtils.executeSqlQuery(sql,{ d->
if(d.name.contains(name) && d.type.contains("住宅")){
total ++
if(d.name.contains(name) && d.type.contains("住宅") && d.shape){
obj = d
}
})
return total;
return obj;
}
@@ -3,7 +3,7 @@ package com.fangjia.script.gj.ccb
import org.apache.commons.lang.StringUtils
class FileUtils {
static def batchCreateFile(ret, fileName, headers,charset, isAppend){
static def batchCreateFile(ret, fileName, headers, charset, isAppend){
File file = new File(fileName);
// if (file.exists()) {
// file.delete();
@@ -51,6 +51,63 @@ class FileUtils {
out.close();
}
static def batchCreateFileByCustomSuffix(ret, fileName, headers, charset, isAppend, suffix, lineSuffix){
File file = new File(fileName);
// if (file.exists()) {
// file.delete();
// }
if (!file.getParentFile().exists()) {
boolean result = file.getParentFile().mkdirs();
if (!result) {
System.out.println("´´½¨Ä¿Â¼Ê§°Ü");
}
}
OutputStream out = new FileOutputStream(fileName, isAppend);
def sb = new StringBuffer();
if(headers){
sb.append(headers.join(suffix))
sb.append("\n");
}
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
def index = 0
for(def row : ret){
if(row){
index++
sb.append(StringUtils.join(row.toArray(), suffix))
if(index<ret.size()){
sb.append(lineSuffix+"\n");
}else{
sb.append(lineSuffix);
}
if(index%10000==0){
println index+"¿ªÊ¼Ð´ÈëCSV "+new Date().toLocaleString()
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
println index+"½áÊøÐ´ÈëCSV "+new Date().toLocaleString()
}
}
}
if(sb.length()>0){
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
}
out.close();
}
static def createTxtFile(ret, fileName, charset, isAppend){
File file = new File(fileName);
if (!file.getParentFile().exists()) {
@@ -0,0 +1,270 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.util.DateUtil
import com.fangjia.script.averager.InsertMonitorCalcCommandBatch
import com.fangjia.script.averager.InsertMonitorSquareRentCalcCommandBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.MongoClient
import com.rabbitmq.client.AMQP
class AddCityRegionPrice {
public final static DBCollection COMMAND_COLL = new MongoClient("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('calc_command')
public static final def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName2("fangjia", "sqmetre_rent", mongoDbUrl)
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", mongoDbUrl)
static main(args) {
def task = new AddCityRegionPrice()
// task.addRegionPrice()
task.addCityprice()
// task.addDistrictprice()
// task.addCityRentPrice()
// task.addRegionRentPrice()
// task.addDistrictRentprice()
// task.updateRegionRentData()
// task.updateDealPriceAreaType()
// task.checkCityImport()
}
def addRegionPrice(){
InsertMonitorCalcCommandBatch insertBatch = new InsertMonitorCalcCommandBatch()
def citys = ["文山州"]//CommonDatautils.findAllCity()
def add = 0
for(def c : citys){
def regions = CommonDatautils.getCityRegions(c)
for(def r : regions){
def startObj = CommonDatautils.queryCityOrRegionPrice(c, r, 1)
def endObj = CommonDatautils.queryCityOrRegionPrice(c, r, -1)
if(startObj && endObj){
def betweenDates = DateUtil.getWeekendBetweenDates(startObj.weekend, endObj.weekend);
for(def weekend : betweenDates ){
def price = CommonDatautils.findCityOrRegionPrice(c, r, weekend)
if(!price){
add++
insertBatch.saveCityCalcCommand(c,r,null, weekend, "fangjia-priceanalysis-0")
println "[add] ${add} "+c+" "+r+" "+DateUtil.formatByPattern(weekend, "yyyy-MM-dd")
}
}
}else{
println c+" "+r+" 有空值"
}
}
}
}
def addCityprice(){
InsertMonitorCalcCommandBatch insertBatch = new InsertMonitorCalcCommandBatch()
def citys = ["乐东"]// CommonDatautils.findAllCity().reverse()
def add = 0
for(def c : citys){
def startObj = CommonDatautils.queryCityOrRegionPrice(c, null, 1)
def endObj = CommonDatautils.queryCityOrRegionPrice(c, null, -1)
if(startObj && endObj){
def betweenDates = DateUtil.getWeekendBetweenDates(startObj.weekend, endObj.weekend);
for(def weekend : betweenDates ){
def price = CommonDatautils.findCityOrRegionPrice(c, null, weekend)
if(!price){
add++
insertBatch.saveCityCalcCommand(c,null,null, weekend, "fangjia-priceanalysis-0")
println "[add] ${add} "+c+" "+DateUtil.formatByPattern(weekend, "yyyy-MM-dd")
}
}
println c +" 校验完成"
}else{
println c+" 有空值"
}
}
}
def addDistrictprice(){
InsertMonitorCalcCommandBatch insertBatch = new InsertMonitorCalcCommandBatch()
def citys = CommonDatautils.findAllCity().reverse()
def add = 0
for(def c : citys){
def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2"])
for(def d : districts){
def startObj = CommonDatautils.queryFirstOrLastDistrictPrice(c, d.region, d.name, 1)
def endObj = CommonDatautils.queryFirstOrLastDistrictPrice(c, d.region, d.name, -1)
if(startObj && endObj){
def betweenDates = DateUtil.getWeekendBetweenDates(startObj.weekend, endObj.weekend);
for(def weekend : betweenDates ){
def price = CommonDatautils.queryDistrictPriceByWeekend(c, d.region, d.name, weekend)
if(!price){
add++
insertBatch.saveDistrictCalcCommand(c, d.region, d.name, d.estate_type2, weekend, "fangjia-priceanalysis-0")
println "[add] ${add} "+c+" "+d.region+" "+d.name+" "+DateUtil.formatByPattern(weekend, "yyyy-MM-dd")
}
}
println c +" 校验完成"
}else{
println c+ " "+d.region+" "+d.name+" 无价格"
}
}
}
}
def addCityRentPrice(){
InsertMonitorSquareRentCalcCommandBatch inserBatch = new InsertMonitorSquareRentCalcCommandBatch();
def citys = CommonDatautils.findAllCity()
for(def c : citys){
def startObj = CommonDatautils.queryFirstOrLastRentPrice(c, null, null, 1)
def endObj = CommonDatautils.queryFirstOrLastRentPrice(c, null, null, -1)
if(startObj && endObj){
def months = DateUtils.getBetweenMonth(DateUtil.formatByPattern(startObj.weekend, "yyyyMM"),"202409")
months.each {m->
def lastDay = DateUtils.truncateDay(DateUtils.getLastDayOfMonth(DateUtil.parseDate(m+"01", "yyyyMMdd")))
def obj = CommonDatautils.findSqmetreRent(c, null, null, lastDay)
if(!obj){
inserBatch.saveCityCalcCommand(c, null, null, lastDay, "fangjia-priceanalysis-0")
println "[add] "+c+" "+DateUtil.formatByPattern(lastDay, "yyyy-MM-dd")
}
}
}
}
}
def addRegionRentPrice(){
InsertMonitorSquareRentCalcCommandBatch inserBatch = new InsertMonitorSquareRentCalcCommandBatch();
def citys = CommonDatautils.findAllCity()
for(def c : citys){
def regions = CommonDatautils.getCityRegions(c)
for(def r : regions){
// if(r!="南川"){
// continue
// }
def startObj = CommonDatautils.queryFirstOrLastRentPrice(c, r, null, 1)
def endObj = CommonDatautils.queryFirstOrLastRentPrice(c, r, null, -1)
if(startObj && endObj){
def months = DateUtils.getBetweenMonth(DateUtil.formatByPattern(startObj.weekend, "yyyyMM"),"202409")
months.each {m->
// println m
def lastDay = DateUtils.truncateDay(DateUtils.getLastDayOfMonth(DateUtil.parseDate(m+"01", "yyyyMMdd")))
def obj = CommonDatautils.findSqmetreRent(c, r, null, lastDay)
if(!obj){
inserBatch.saveCityCalcCommand(c, r, null, lastDay, "fangjia-priceanalysis-0")
println "[add] "+c+" "+r+" "+DateUtil.formatByPattern(lastDay, "yyyy-MM-dd")
}
}
}
}
}
}
def addDistrictRentprice(){
InsertMonitorSquareRentCalcCommandBatch inserBatch = new InsertMonitorSquareRentCalcCommandBatch();
def citys = CommonDatautils.findAllCity()
for(def c : citys){
def buffer = []
def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2"])
for(def d : districts){
def startObj = CommonDatautils.queryFirstOrLastRentPrice(c, d.region, d.name, 1)
// def endObj = CommonDatautils.queryFirstOrLastRentPrice(c, d.region, d.name, -1)
if(startObj){
def months = DateUtils.getBetweenMonth(DateUtil.formatByPattern(startObj.weekend, "yyyyMM"),"202409")
months.each {m->
def lastDay = DateUtils.truncateDay(DateUtils.getLastDayOfMonth(DateUtil.parseDate(m+"01", "yyyyMMdd")))
def obj = CommonDatautils.findSqmetreRent(c, d.region, d.name, lastDay)
if(!obj){
def cmd = inserBatch.saveDistrictCalcCommand(c, d.region, d.name, d.estate_type2, lastDay, "fangjia-priceanalysis-0")
buffer << cmd
if(buffer.size() == 1000){
COMMAND_COLL.insert(buffer)
buffer.clear()
}
println "[add] "+c+" "+ d.name+" "+DateUtil.formatByPattern(lastDay, "yyyy-MM-dd")
}
}
}
}
if(!buffer.isEmpty()){
COMMAND_COLL.insert(buffer)
}
}
}
def updateRegionRentData(){
def q = new BasicDBObject("cat", "region")
def keys = new BasicDBObject()
keys.put("markup", 1)
def dbcursor = SQMETRE_RENT_COLL.find(q, keys).batchSize(5000)
def index = 1
for(def d : dbcursor){
if(!(d.markup instanceof Double)){
def udata = new BasicDBObject()
double markup = 1.0
markup = d.markup * markup
udata.put("markup", markup)
SQMETRE_RENT_COLL.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
println index
index++
}
}
}
def updateDealPriceAreaType(){
def index = 1
def citys = ["凉山州","福州","南京","佛山","沈阳","苏州"]
citys.each {c->
def q = new BasicDBObject()
q.put("city", c)
// q.put("price_flag", "国信达成交-推导")
def dbcursor = DEAL_PRICE_COLL.find(q)
for(def d : dbcursor){
// if(!(d.area instanceof Double)){
// def udata = new BasicDBObject()
// double area = 1.0
// area = d.area * area
// udata.put("area", area)
// DEAL_PRICE_COLL.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
// println c+" "+index
// index++
// }
if(!(d.total_price instanceof Integer)){
def udata = new BasicDBObject()
udata.put("total_price", d.total_price.toString().toDouble().toInteger())
DEAL_PRICE_COLL.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
println c+" "+index
index++
}
}
}
}
def checkCityImport(){
def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]
for(def c : citys){
def startObj = CommonDatautils.findStatIndexBySort("城市", c, null, 1)
def endObj = CommonDatautils.findStatIndexBySort("城市", c,null, -1)
if(startObj && endObj){
def months = DateUtils.getBetweenMonth(startObj.yearmonth.toString(),"202409")
months.each {m->
def obj = CommonDatautils.findStatIndexByMonth("城市", c,null, m.toInteger())
if(!obj){
println "[add] "+c+" "+m
}
}
}else{
println c +" 一个也没有"
}
}
}
}
@@ -0,0 +1,376 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import groovy.json.JsonSlurper
import net.sf.json.JSONObject
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 添加高德地图,百度地图围栏
*/
@BatchOwner(Watcher.GaoJuan)
class AddDistrictShapeBatch extends BaseBatch{
def fullCityMap = CommonDatautils.queryFullNameCity()
def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
runBatch(new AddDistrictShapeBatch(), args)
}
@Override
public void process(Object args) {
// findGaoDeDatas2("苏州市","吴中区", "水岸清华瀚宫" )
def citys = CommonDatautils.findAllCity()
def index = 1
for(def c : citys){
println c +" "+index
index ++
def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2"])
// baiduMatch(c, districts)
gaodeMatch(c, districts)
}
}
def baiduMatch(city,districts){
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
fullCityName = data.city
}
final CountDownLatch latch = new CountDownLatch(districts.size())
if(fullCityName){
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
for(def i=0;i<districts.size();i++){
final def d = districts[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def baiduId = baiduProcess(d, fullCityName)
if(baiduId){
if(!rets.contains(baiduId)){
rets << baiduId
}else{
println "百度围栏 "+baiduId +" 被:["+d.city+" "+d.region+" "+d.name+" ]再次匹配,查看原始小区是否重复或别名相同"
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
}else{
println city +" 找不到相应的全称"
}
latch.await()
}
def baiduProcess(d, fullCityName){
def alias = []
alias << d.name
if(d.alias){
alias.addAll(d.alias)
}
def distData = mcaDistMap.get(d.city+"#"+d.region)
def fullDistName = null
if(distData){
fullDistName = distData.district
}
def fenceMap = null
for(def name : alias){
fenceMap = findBaiduDatas(fullCityName, fullDistName, name)
if(!fenceMap){
continue
}
}
if(fenceMap){
def o = new BasicDBObject()
o.put("city", d.city)
o.put("district", d.region)
o.put("name", d.name)
o.put("source", "百度地图")
o.put("c_date", new Date())
o.put("data_id", fenceMap.data_id)
o.put("comm_id", d._id.toString())
def resultMap = [:]
resultMap.coordinates = fenceMap.coordinatePairs
// if(fenceMap.coordinatePairs.size()>1) {
// resultMap.type = "MultiPolygon"
// }else{
resultMap.type = "polygon"
// }
o.put("shape", resultMap)
def q = new BasicDBObject("comm_id", d._id.toString())
seaweed_fence_coll.update(q, new BasicDBObject('$set', o),true,false)
// seaweed_fence_coll.save(o)
println "[save] 围栏 ( "+d.city +" "+d.region +" "+d.name+" )"
return fenceMap.data_id
}
return null
}
def findBaiduDatas(city, district, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("district", district)
q.put("name", name)
q.put('std_tag', '房地产;住宅区')
q.put("guoke_geo", new BasicDBObject('$ne', null))
def dbcursor = BAIDU_POI_COLL.find(q)
def map = [:]
println dbcursor.count()
for(def d : dbcursor){
def arrs = d.gcj02_guoke_geo.split("\\|")
def coordinatesStr = arrs[2].toString().substring(2, arrs[2].toString().size()-1 )
// 将字符串分割成列表
def coordinatePairs = coordinatesStr.split(',')
def coordinatesList = []
for (int i = 0; i < coordinatePairs.size(); i += 2) {
// 将字符串坐标转换为浮点数
double longitude = coordinatePairs[i] as double
double latitude = coordinatePairs[i + 1] as double
// 将坐标点作为一个元组添加到集合中
coordinatesList << [longitude, latitude]
}
if(coordinatePairs){
map.data_id = d._id
map.coordinatePairs = coordinatesList
}
}
return map;
}
def gaodeMatch(city,districts){
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
fullCityName = data.city
}
final CountDownLatch latch = new CountDownLatch(districts.size())
if(fullCityName){
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
for(def i=0;i<districts.size();i++){
final def d = districts[i]
def fence = findSeaweedFenceExists(d._id.toString())
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
if(!fence){
def baiduId = gaodeProcess(d, fullCityName)
if(baiduId){
if(!rets.contains(baiduId)){
rets << baiduId
}else{
println "高德围栏 "+baiduId +" 被:["+d.city+" "+d.region+" "+d.name+" ]再次匹配,查看原始小区是否重复或别名相同"
}
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
}else{
println city +" 找不到相应的全称"
}
latch.await()
}
def gaodeProcess(d, fullCityName){
def alias = []
alias << d.name
if(d.alias){
alias.addAll(d.alias)
}
def distData = mcaDistMap.get(d.city+"#"+d.region)
def fullDistName = null
if(distData){
fullDistName = distData.district
}
def fenceMap = null
for(def name : alias){
fenceMap = findGaoDeDatas2(fullCityName, fullDistName, name)
if(!fenceMap){
continue
}
}
if(fenceMap){
def o = new BasicDBObject()
o.put("city", d.city)
o.put("district", d.region)
o.put("name", d.name)
o.put("source", "高德地图")
o.put("c_date", new Date())
o.put("data_id", fenceMap.id)
o.put("comm_id", d._id.toString())
def resultMap = [:]
// def shape = fenceMap.shape
// def points_x = fenceMap.shape.polygon.x
// def points_y = fenceMap.shape.polygon.y
// def list = []
// for(def i=0;i<points_x.size();i++){
// list << [points_x[i], points_y[i]]
// }
resultMap.coordinates = fenceMap.coordinates
resultMap.type = "polygon"
o.put("shape", resultMap)
def q = new BasicDBObject("comm_id", d._id.toString())
seaweed_fence_coll.update(q, new BasicDBObject('$set', o),true,false)
// seaweed_fence_coll.save(o)
println "[save] 高德围栏 ( "+d.city +" "+d.region +" "+d.name+" )"
return fenceMap.data_id
}
return null
}
def findGaoDeDatas(city, region, name){
String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
def obj = null
ElasticUtils.executeSqlQuery(sql,{ d->
def newName = d.name.replace("-","").replace("·","")
if(newName == name && d.type.contains("住宅") && d.shape){
obj = d
}
})
return obj;
}
def findGaoDeDatas2(city, region, name){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:goojia123456".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.206:9200/amap_poi_detail/_search"
def body = """
{
"_source": {
"includes": ["name", "shape", "id"]
},
"query": {
"bool": {
"must": [
{
"term": {
"cityname" : "${city}"
}
},
{
"term": {
"adname" : "${region}"
}
},
{
"match": {
"name": {"query": "${name}","operator": "and"}
}
},
{
"match": {
"type": "住宅"
}
},
{
"exists": {
"field": "shape"
}
}
]
}
}
}
"""
HttpRequest request = new HttpRequest()
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = new JsonSlurper().parseText(res)
def obj = null
def fenceMap = [:]
if(response && response.hits){
def hits = response.hits.hits
if(hits){
def d = hits[0]
def source = d._source
def newName = source.name.replace("-","").replace("·","").replace("(建设中)","")
if(newName == name){
fenceMap.id = source.id
def list = []
def points_arr = source.shape.coordinates
for(def points : points_arr){
def rets = []
for(def point : points){
def lng = DataHandleUtils.getDoubleRound(6, point[0].toString().toDouble())
def lat = DataHandleUtils.getDoubleRound(6, point[1].toString().toDouble())
rets << [lng, lat]
}
list << rets
}
if(list.size()==1){
fenceMap.coordinates = list[0]
}else{
fenceMap.coordinates = list
}
}
}
}
return fenceMap
}
def findSeaweedFenceExists(commid){
def q = new BasicDBObject()
q.put("comm_id", commid)
return seaweed_fence_coll.findOne(q)
}
}
@@ -0,0 +1,234 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 小区自动入库(新增小区,每月运行一次)
*/
class AutoAddDistrictBatch extends BaseBatch{
final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018)
public static DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def fullCityMap = CommonDatautils.queryFullNameCity()
def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
static main(args) {
runBatch(new AutoAddDistrictBatch(), args)
}
@Override
void process(Object args) {
//查找第三方匹配没有匹配上的小区
def citys = ["上海"]//CommonDatautils.findAllCity()
def list = []
list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "高德地图"]
for(def c : citys){
def noMatchDatas = findNoMatchResult(c)
println c +" 新旧算法未匹配的数据量:"+noMatchDatas.size()
def rets = checkDatas(noMatchDatas.keySet())
list.addAll(rets)
}
def fileName = "/home/goojia/gj/add_dist_info.xlsx"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\小区\\第三方待入库数据.xlsx"
}
ExcelUtils.write(fileName, list, "待入库数据")
}
def checkDatas(keysets){
def noMatchCount = keysets.size()
final CountDownLatch latch = new CountDownLatch(noMatchCount)
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
for(def i=0;i<keysets.size();i++){
final def key = keysets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def arrs = key.split("##")
//检测有没有成交
def trade_count = findTrades(arrs[0], arrs[1], arrs[2])
//检测有没有房源
def house_count = CommonDatautils.findHouseCount(arrs[0], arrs[1], arrs[2])
//检查百度地图有没有
def baidu_count = findBaiduCount(arrs[0], arrs[1], arrs[2])
//检查高德地图有没有
def gaode_count = findGaodeCount(arrs[0], arrs[1], arrs[2])
println "[ "+totalCount.get()+"/"+noMatchCount+" ]"+key +" trade_count:"+trade_count+" house_count:"+house_count+" baidu_count:"+baidu_count+" gaode_count:"+gaode_count
rets << [arrs[0], arrs[1], arrs[2], trade_count, house_count, baidu_count, gaode_count]
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets
}
def findBaiduCount(city, region, name){
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
fullCityName = data.city
}
if(fullCityName){
def distData = mcaDistMap.get(city+"#"+region)
def fullDistName = null
if(distData){
fullDistName = distData.district
}
if(fullDistName){
return findBaiduDatas(fullCityName, fullDistName, name)
}
}
return null
}
def findGaodeCount(city, region, name){
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
fullCityName = data.city
}
if(fullCityName){
def distData = mcaDistMap.get(city+"#"+region)
def fullDistName = null
if(distData){
fullDistName = distData.district
}
if(fullDistName){
def obj = findGaoDeDatas(fullCityName, fullDistName, name)
if(obj){
return 1
}
}
}
return 0
}
def findGaoDeDatas(city, region, name){
String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
def obj = null
ElasticUtils.executeSqlQuery(sql,{ d->
if(d.name == name && d.type.contains("住宅") && d.shape){
obj = d
}
})
return obj;
}
def findTrades(city, region, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("district_name", name)
return DEAL_PRICE_COLL.count(q)
}
def findNoMatchResult(def c){
def q = new BasicDBObject()
q.put("match_id", new BasicDBObject('$exists',false))
q.put("old_match_id", new BasicDBObject('$exists',false))
q.put("city", c)
def dbcursor = thirdSeaweedMappingColl.find(q).batchSize(5000)
def rMap = [:]
for(def d : dbcursor){
if(d.estate_type){
if(d.estate_type.contains("自建房")){
continue
}
if(d.estate_type.contains("购物中心")){
continue
}
if(d.estate_type.contains("住宅底商") || d.estate_type.contains("酒店") || d.estate_type.contains("商铺")){
continue
}
if(d.estate_type.contains("办公") || d.estate_type.contains("写字楼")){
continue
}
}else{
continue
}
def key = d.city + "##"+d.district+"##"+d.name
def rets = rMap.get(key)
if(!rMap.containsKey(key)){
rets = []
rMap.put(key, rets)
}
rets << d
}
return rMap
}
def createData(city, region, name, address, lng, lat){
def udata = new BasicDBObject()
udata.put("city", city)
udata.put("region", region)
name = name.replaceAll('[·\\(\\)()]|·', '').replace("建设中", "")
udata.put("name", name)
def addr = AddressUtil.formatAddress(city, address)
if(addr){
udata.put("address", addr)
}
udata.put("status", "0")
udata.put("name_cat", "0")
udata.put("cat", "district")
udata.put("username", "高娟")
def estateType2 = "普通住宅"
def type = ""
if(type){
if(type.contains("别墅")){
estateType2 = "别墅"
}else if(type.contains("商住")){
estateType2 = "商住"
}
}
udata.put("estateType2", estateType2)
if(lng){
Long gd_lng = lng*10000000000
udata.put("lng", gd_lng)
}
if(lat){
Long gd_lat = lat * 10000000000
udata.put("lat", gd_lat)
}
FangjiaClientUtils.saveOrUpdateDistrict(udata)
}
def findBaiduDatas(city, district, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("district", district)
q.put("name", name)
q.put('std_tag', '房地产;住宅区')
def count = BAIDU_POI_COLL.count(q)
return count;
}
}
@@ -17,7 +17,9 @@ import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.poi.ss.util.RegionUtil
import org.bson.types.ObjectId
import org.joda.time.Instant
import java.time.format.DateTimeFormatter
import java.util.regex.Pattern
class DevProcess {
@@ -35,11 +37,16 @@ class DevProcess {
// DBCollection kfq_COLL = MongoUtils.getCollectionByName("crawl", "kfq_investchn_com", "192.168.0.206",27020)
DBCollection TOWNPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
final DBCollection tempJiangsuQiliangColl = MongoUtils.getCollectionByName("crawl", "temp_jiangsu_qiliang", "192.168.0.206",27020)
static main(args) {
def task = new DevProcess();
// task.run()
// task.parseCityAndDist()
task.devSplitRoad()
// task.devSplitRoad()
// task.devNameSplitRoad()
// task.addFlag()
// task.matchDev()
@@ -56,7 +63,7 @@ class DevProcess {
// task.createDevMapping()
// task.deleteAddr()
// task.deleteAddr2()
// task.synchDevTo100()
task.synchDevTo100()
// task.addNewDev2()
// task.addAlias()
@@ -75,10 +82,16 @@ class DevProcess {
// task.repeatSave()
// task.addDevInfo()
// task.matchDevShape()//匹配开发区围栏
// task.addQiDevResult() //获取齐梁数据结果
}
def parseCityAndDist(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\等级开发区.xlsx",{d->
if(!d['J']){
@@ -271,30 +284,30 @@ class DevProcess {
while(i<20){
def fileName = "/home/goojia/gj/qzd-${i}.xlsx"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\数据建设\\开发区\\新跑省份\\河北省\\test-${i}.xlsx"
fileName = "D:\\数据\\数据建设\\开发区\\乡镇街道围栏开发区数据.xlsx"
}
File file=new File(fileName);
if(file.exists()){
ExcelUtils.multiThreadingModify(fileName,{d->
if(d['C'] && !d['F']){
def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['C'])
if(jsonObject1) {
if (jsonObject1.get("dev") && jsonObject1.get("dev") != "null") {
def devName = jsonObject1.get("dev")
def obj = findDev(devName)
d['F'] = devName
if(jsonObject1.containsKey("detail") && jsonObject1.detail.dev_confi!='null'){
d['S'] = jsonObject1.detail.dev_confi
}
if (obj) {
d['O'] = obj._id.toString()
d['P'] = obj.name
d['U'] = obj.match_way
}
}
}
}
// if(d['C'] && !d['F']){
// def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['C'])
// if(jsonObject1) {
// if (jsonObject1.get("dev") && jsonObject1.get("dev") != "null") {
// def devName = jsonObject1.get("dev")
// def obj = findDev(devName)
// d['F'] = devName
// if(jsonObject1.containsKey("detail") && jsonObject1.detail.dev_confi!='null'){
// d['S'] = jsonObject1.detail.dev_confi
// }
// if (obj) {
// d['O'] = obj._id.toString()
// d['P'] = obj.name
// d['U'] = obj.match_way
// }
// }
// }
// }
if(d['E'] && !d['M']) {
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['E'])
@@ -326,7 +339,7 @@ class DevProcess {
d['N'] = "信息不完整"
}
if (!d['N'] && d['C'] && !d['C'].contains(road)) {
if (!d['N'] && !d['E'].contains(road)) {
d['N'] = "原地址不含路"
}
@@ -429,20 +442,20 @@ class DevProcess {
}
def matchDev(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\前瞻网开发区对应园区.xlsx",{d->
if(!d['I'] && d['J']){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\新跑省份\\江苏省\\江苏省-齐梁待跑地址.xlsx",{d->
if(!d['B'] && d['A']){
def q = new BasicDBObject()
// q.put("name", d['J'])
// q.put("name", d['A'])
// q.put("city",d['G'])
// q.put('alias', new BasicDBObject('$in',[d['J']]))
def name = d['J'].replace('经济技术开发区','经济开发区')
// q.put('alias', new BasicDBObject('$in',[d['A']]))
def name = d['A'].replace('经济技术开发区','经济开发区')
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
def obj = development_zone_CONN.findOne(q)
if(obj){
d['I'] = "库中有开发区-相似2"
d['K'] = obj.name
// d['K'] = obj.name
// d['C'] = obj._id.toString()
// d['D'] = obj.name
d['B'] = obj.name
// if(obj.name == d['A']){
// d['E'] = "相同"
// }else{
@@ -452,7 +465,7 @@ class DevProcess {
// d['F'] = obj.name
}
}
},0);
},1);
}
def calAddrDistince(){
@@ -619,7 +632,7 @@ class DevProcess {
def addDevAddres3(){
def index = 0
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\新跑省份\\新疆省入库.xlsx",{d->
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\新跑省份\\河北省\\河北省开发区地址入库.xlsx",{d->
if(d['A']){
def obj = development_zone_CONN.findOne(new BasicDBObject("name", d['A']))
if(obj){
@@ -1063,4 +1076,76 @@ class DevProcess {
}
DEV_MAPPING_CONN2.insert(list)
}
def addDevInfo(){
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Downloads\\河北省开发区对应园区.xlsx",{d->
if(d['C']){
def q = new BasicDBObject().append("name", d['C'])
def obj = development_zone_CONN.findOne(q)
if(obj){
d['A'] = obj.city
d['B'] = obj.district
}
}
})
}
def matchDevShape(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\开发区-全量9-14.xlsx",{d->
if(d['C'] && !d['I']){
def newDev = d['E'].replace("经济技术开发区", "经济开发区").replace(d['B'].replace("省",""), "")
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(new BasicDBObject("city",d['C']).append("district", d['D']).append("type","乡镇街道").append("street", new BasicDBObject('$regex', Pattern.compile(newDev.trim()+'.*$', Pattern.CASE_INSENSITIVE))).append("shape", new BasicDBObject('$exists',true)))
if(dbcursor.count()>0){
println "[统计局] "+d['E'] +" "+ dbcursor.count()
if(dbcursor.count()==1){
for(def obj : dbcursor){
d['F'] = obj.city
d['G'] = obj.district
d['H'] = obj.street
d['I'] = "统计局相似(去省)-有围栏"
}
}
}else{
dbcursor = TOWNPOLYLINE_COLL.find(new BasicDBObject("city",d['C']).append("region", d['D']).append("town", newDev))
println "[乡镇街道围栏] "+d['E'] +" "+ dbcursor.count()
if(dbcursor.count()==1){
for(def obj : dbcursor){
d['F'] = obj.city
d['G'] = obj.region
d['H'] = obj.town
if(obj.polyline){
d['I'] = "乡镇围栏相似(去省)-有围栏"
}
}
}
}
}
});
}
def addQiDevResult(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\新跑省份\\江苏省\\江苏省-齐梁待跑地址.xlsx",{d->
if(d['E'] && !d['Z']){
def q = new BasicDBObject("qiye_address", d['E'])
def obj = tempJiangsuQiliangColl.findOne(q)
if(obj){
def qlmap = obj.ql
if(qlmap){
d['Z'] = "有"
if(qlmap.indusParkList){
def parklist = qlmap.indusParkList
def names = []
for(def p : parklist){
names << p.indusParkName
}
d['W'] = names.join(",")
d['X'] = parklist.size()
}
}
}
}
},0)
}
}
@@ -19,7 +19,11 @@ import org.elasticsearch.common.xcontent.XContentType
class DevUpdateTestBatch {
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection development_zone_43_CONN = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
DBCollection development_zone_100_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
DBCollection DEV_MAPPING_100_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
@@ -31,8 +35,8 @@ class DevUpdateTestBatch {
def task = new DevUpdateTestBatch()
def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx
// task.addDev(path)
// task.addDevAddress(path)
task.addDevAddress(path)
//
// task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是")
@@ -42,7 +46,7 @@ class DevUpdateTestBatch {
def addDev(path){
ExcelUtils.modify(path,{ d->
if(!d['I'] && d['A']){
def obj = development_zone_100_CONN.findOne(new BasicDBObject("name", d['A']))
def obj = development_zone_43_CONN.findOne(new BasicDBObject("name", d['A']))
if(!obj){
def o = new BasicDBObject()
o.put("province", d['B'])
@@ -109,7 +113,7 @@ class DevUpdateTestBatch {
def addDevAddress(path){
ExcelUtils.modify(path,{ d->
if(!d['D'] && d['A']){
def obj = development_zone_100_CONN.findOne(new BasicDBObject("name", d['A']))
def obj = development_zone_43_CONN.findOne(new BasicDBObject("name", d['A']))
if(obj){
def q = new BasicDBObject("name", d['A']).append("type",d['B'])
if(d['B'] == "乡镇街道"){
@@ -118,6 +122,8 @@ class DevUpdateTestBatch {
q.put("address", d['C'])
}else if(d['B'] == "园区"){
q.put("park", d['C'])
}else if(d['B'] == "Poi"){
q.put("poi", d['C'])
}
def mappingObj = DEV_MAPPING_100_CONN.findOne(q)
if(!mappingObj){
@@ -133,6 +139,8 @@ class DevUpdateTestBatch {
o.put("address", d['C'])
}else if(d['B'] == "园区"){
o.put("park", d['C'])
}else if(d['B'] == "Poi"){
o.put("poi", d['C'])
}
def date = new Date()
o.put("c_date", date)
@@ -140,15 +148,15 @@ class DevUpdateTestBatch {
DEV_MAPPING_100_CONN.save(o)
// o.id = o._id.toString()
// o.remove("_id")
// o.remove("c_date")
// addEs(o, "192.168.0.89", "dev_mapping")
// try{
// addEs(o, "192.168.0.52", "dev_mapping")
// }catch(Exception e){
// println "52 es error: "+e.printStackTrace()
// }
o.id = o._id.toString()
o.remove("_id")
o.remove("c_date")
addEs(o, "192.168.0.89", "dev_mapping")
try{
addEs(o, "192.168.0.52", "dev_mapping")
}catch(Exception e){
println "52 es error: "+e.printStackTrace()
}
d['D'] = "已完成"
}else{
@@ -160,7 +168,7 @@ class DevUpdateTestBatch {
println d['A'] +" 不存在"
}
}
},1)
},2)
}
@@ -28,7 +28,7 @@ class GXDdistrictMatch {
static main(args) {
def task = new GXDdistrictMatch();
// task.run()
task.addOldMatch()
task.addOldMatch2()
// task.runDb()
// task.updateDistStatus()//ťÖ¸´name_cat˛ťŐýłŁĘýžÝ
@@ -40,10 +40,40 @@ class GXDdistrictMatch {
def addOldMatch(){
//ŔĎĆĽĹä
def fileName = "D:\\数据\\建行\\房产估值\\20240716-20240815 _业务反馈价格.xlsx"//11城重新匹配.xlsx
def fileName = "/home/goojia/gj/map.xlsx"
if (CommonDatautils.isWindow()) {
fileName = "C:\\Users\\gaojuan\\Downloads\\百度有围栏数据1.xlsx"//11城重新匹配.xlsx
}
ExcelUtils.multiThreadingModify(fileName, { d ->
println d['C']
if(!d['AJ'] && d['I']){
if(!d['AJ'] && d['C']){
def jsonObject = OpenApiClient.addressMatch(d['A'], d['D'], "district", false,d['B'], d['C'])
if (jsonObject != null && !jsonObject.containsKey("result")) {
d['AJ'] = jsonObject.get("id")
d['AK'] = jsonObject.get("city")
d['AL'] = jsonObject.get("district")
d['AM'] = jsonObject.get("name")
d['AN'] = jsonObject.get("address")
def dbo = CommonDatautils.findDistrictById(jsonObject.get("id"))
if(dbo){
d['AO'] = dbo.status
d['AP'] = dbo.name_cat
d['AQ'] = dbo.estate_type2
}
}
}
});
}
def addOldMatch2(){
//老匹配
def fileName = "D:\\数据\\建行\\房产估值\\建行业务反馈badcase.xlsx"//11城重新匹配.xlsx
ExcelUtils.multiThreadingModify(fileName, { d ->
println d['C']
if(!d['AJ']){
def jsonObject = OpenApiClient.addressMatch(d['C'], d['D'], "district", false,null, d['D'])
if (jsonObject != null && !jsonObject.containsKey("result")) {
d['AJ'] = jsonObject.get("id")
@@ -36,8 +36,8 @@ class MinglujiData {
// exportDatas(city)
// matchDev(prov, city)
// matchCommunity(city)
// fiveLevelSplit(city)
nineLevelSplit(city)
fiveLevelSplit(city)
// nineLevelSplit(city)
// tempMatchDev(prov)
// checkDev()
@@ -214,9 +214,10 @@ class MinglujiData {
fileName = "D:\\Êý¾Ý\\Êý¾Ý½¨Éè\\Ãû¼¼¯\\${city}_${i}.xlsx"
}
println "开始读取:"+fileName
File file = new File(fileName);
if (file.exists()) {
ExcelUtils.modify(fileName,{d->
ExcelUtils.multiThreadingModify(fileName,{d->
if(d['E'] && !d['Z']){
def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['E'])
if(jsonObject1) {
@@ -21,11 +21,12 @@ import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
class PoiLocation {
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "didata_map_poi_single", MAP_POI_URL)
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
def regionMap = com.fangjia.script.commData.CommonDatautils.queryDistNameByCity()
@@ -35,14 +36,14 @@ class PoiLocation {
static main(args) {
def task = new PoiLocation();
// task.run();
task.regionAndStreetProcess()
task.run();
// task.regionAndStreetProcess()
}
def run(){
def citys = [""]
def citys = ["深圳市"]
citys = citys.reverse()
for(def city : citys){
findPois(city)
@@ -80,6 +81,8 @@ class PoiLocation {
def findPois(city){
def q = new BasicDBObject()
q.put("city", city)
q.put("district", "坪山区")
q.put('address',new BasicDBObject('$regex', Pattern.compile('^.*兰竹东路8号.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
// q.put("run_status", new BasicDBObject('$ne', 1))
def keys = new BasicDBObject()
@@ -151,8 +154,8 @@ class PoiLocation {
for(def i=0;i<datas.size();i++){
final def d = datas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
// fixedThreadPool.execute(new Runnable() {
// void run() {
try{
totalCount.addAndGet(1)
def map = running(d)
@@ -170,8 +173,8 @@ class PoiLocation {
} finally {
latch.countDown()
}
}
});
// }
// });
}
latch.await()
return rets
@@ -198,6 +201,8 @@ class PoiLocation {
if(d.township){
town = getSimilarTown(regions, d.city, d.township)
}
def newTown = getTownByLocation(d.city, regions, d.location)
if(town){
if(town.street == d.township){
//´ò±ê¼Ç
@@ -311,6 +316,27 @@ class PoiLocation {
return town
}
def getTownByLocation(city,regions, location){
def town = null
def townFlag = null
for(def r : regions){
def townList = commMap.get(city+"#"+r.district)
for(def townObj : townList){
if (townObj.shape && townObj.shape.coordinates) {
townFlag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(townObj.shape.coordinates))
if(townFlag){
town = townObj
break
}
}
}
if(townFlag){
break
}
}
return town
}
def updateMapPoi(def id, def township){
def header = [:]
@@ -0,0 +1,370 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.update.UpdateRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
/**
* 区域标准化
* 1.小区库区域标准化,包含小区,商铺,写字楼
* 2.poi数据库中 区域标准化
*/
class RegionStandardBatch extends BaseBatch{
def fullCityMap = CommonDatautils.queryFullNameCity()
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
static main(args) {
runBatch(new RegionStandardBatch(), args)
}
@Override
void process(Object args) {
// seaweedRegionStandard()
poiRegionStandard()
}
def seaweedRegionStandard(){
seaweedRegionStandardByCat("district")
}
def poiRegionStandard(){
def runCitys = ["重庆","三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德"]
def index = 0
def citys = ["攀枝花"]//CommonDatautils.findAllCity()
for(def city : citys){
index ++
// if(index<32){
// continue
// }
if(runCitys.contains(city)){
continue
}
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
fullCityName = data.city
}
if(fullCityName){
def rMap = regionFullNameMap.get(fullCityName)
if(rMap){
def set = rMap.keySet()
if(set.size()>1){
def rets = findEsPoiByCity(fullCityName)
println city +" "+rets.size()
poiHaveRegionProcess(city, set, rMap, rets)
}else{
// haveStreetProcess(city, set, cat)
}
}else{
println "没找区域:"+city
}
}else{
println "没找到相应的全称:"+city
}
}
}
def seaweedRegionStandardByCat(cat){
def citys = CommonDatautils.findAllCity()
for(def city : citys){
def rMap = regionMap.get(city)
if(rMap){
def set = rMap.keySet()
if(set.size()>1){
haveRegionProcess(city, set, rMap, cat)
}else{
haveStreetProcess(city, set, cat)
}
}
}
}
def haveRegionProcess(city, set, rMap, cat){
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name"])
for(def d : districts){
if(set.contains(d.region)){
continue
}
def updateRegion = null
for(def key : set){
def alias = rMap.get(key)
if(alias.contains(d.region)){
updateRegion = key
break;
}
}
if(updateRegion){
def udata = new BasicDBObject()
udata.put("region", updateRegion)
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name +" 区域修改为:"+updateRegion
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
}else{
def udata = new BasicDBObject()
udata.put("status", 1)
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name +" 状态修改为:删除"
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
}
}
}
def haveStreetProcess(city, set, cat){
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
fullCityName = data.city
}
def streetMap = CommonDatautils.queryAllStreetNameByCity(fullCityName)
if(streetMap){
set = streetMap.keySet()
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name"])
for(def d : districts){
if(set.contains(d.region)){
continue
}
def updateRegion = null
def flag = null
for(def key : set){
def alias = streetMap.get(key)
if((alias && alias.contains(d.region)) || (!alias && key.contains(d.region))){
def newRegion = RegionData.formatRegion(d.city, key)
if(newRegion!=d.region){
updateRegion = newRegion
}
flag = "pass"
break;
}
}
if(updateRegion){
def udata = new BasicDBObject()
udata.put("region", updateRegion)
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name +" 区域修改为:"+updateRegion
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
}else{
if(flag!="pass"){
def udata = new BasicDBObject()
udata.put("status", 1)
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name +" 状态修改为:删除"
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
}
}
}
}
}
def findEsPoiByCity(city){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
def rets = []
String[] includes = ["city", "district", "name", "township", "status", "id"]
def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi", queryBuilder,includes)
for(def d: datas){
def o = new BasicDBObject()
o.put("city", d.city)
o.put("district", d.district)
o.put("name", d.name)
o.put("township", d.township)
o.put("location", d.location)
o.put("status", d.status)
o.put("id", d.id)
rets << o
}
return rets
}
def poiHaveRegionProcess(city, set, rMap, datas){
def results = []
for(def d : datas){
def tjjObj = rMap.get(d.district)
if(tjjObj && tjjObj.mzb_district_name && tjjObj.mzb_district_name==d.district){
continue
}
def updateRegion = null
def updateFlag = true
if(tjjObj){
if(tjjObj.mzb_district_name && tjjObj.mzb_district_name!=d.district){
updateRegion = tjjObj.mzb_district_name
}
}else{
for(def key : set){
def tempObj = rMap.get(key)
def alias = tempObj.alias
if(alias.contains(d.district) || d.district==tempObj.mzb_district_name){
if(d.district!=tempObj.mzb_district_name){
updateRegion = key
}else{
updateFlag = false
}
break;
}
}
}
if(updateFlag){
if(updateRegion){
def m = [:]
m.put(d.id, ["district":updateRegion])
results << m
if(results.size()%100==0){
updateRegionByEs(results)
results.clear()
}
println d.city +" "+d.district+" "+d.name +" "+d.status+" 区域修改为:"+updateRegion
}else{
def m = [:]
m.put(d.id, ["status":1])
results << m
if(results.size()%100==0){
updateRegionByEs(results)
results.clear()
}
println d.city +" "+d.district+" "+d.name +" "+d.status+" 状态修改为:删除"
}
}
}
if(results.size()>0){
updateRegionByEs(results)
results.clear()
}
println city +" 处理完成"
}
def updateRegionByEs(rows){
def client = ElasticUtils89.getClient("192.168.0.89")
def bulkRequest = new BulkRequest()
rows.each { row ->
def id = row.keySet().iterator().next()
def doc = row.get(id)
def updateRequest = new UpdateRequest("didata_map_poi", id)
updateRequest.doc(doc, XContentType.JSON)
bulkRequest.add(updateRequest)
}
try {
// 执行批量更新请求
def bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT)
// 检查响应
if (bulkResponse.hasFailures()) {
println("批量更新中有失败的操作:")
bulkResponse.buildFailureMessage().each { println(it) }
} else {
println("批量更新成功!")
}
} catch (Exception e) {
e.printStackTrace()
} finally {
// 关闭客户端
try {
client.close()
} catch (Exception e) {
e.printStackTrace()
}
}
}
// def queryPoiDatasByCity(){
// def esUrl = "http://localhost:9200/your_index/_search"
// def pageSize = 1000 // 每页返回的记录数
// def scrollSize = '1m' // scroll上下文的生存时间
//
//// 初始化HttpClient和ObjectMapper
// CloseableHttpClient httpClient = HttpClients.createDefault()
// ObjectMapper objectMapper = new ObjectMapper()
//
//// 初始化scroll请求
// def scrollRequest = """
//{
// "size": $pageSize,
// "query": {
// "match_all": {}
// },
// "scroll": "$scrollSize"
//}
//"""
//
//// 发送scroll初始化请求
// HttpGet scrollInitRequest = new HttpGet(esUrl)
// scrollInitRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequest, "UTF-8"))
// scrollInitRequest.setHeader("Content-Type", "application/json")
//
// HttpResponse scrollInitResponse = httpClient.execute(scrollInitRequest)
// HttpEntity entity = scrollInitResponse.getEntity()
// String scrollId = objectMapper.readValue(EntityUtils.toString(entity), Map.class).get("_scroll_id").toString()
//
//// 存储所有结果
// def allResults = []
//
//// 发送scroll请求获取数据,直到没有更多数据为止
// while (true) {
// def scrollRequestWithId = """
// {
// "scroll_id": "$scrollId",
// "scroll": "$scrollSize"
// }
// """
//
// HttpGet scrollGetRequest = new HttpGet(esUrl + "?scroll")
// scrollGetRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequestWithId, "UTF-8"))
// scrollGetRequest.setHeader("Content-Type", "application/json")
//
// HttpResponse scrollGetResponse = httpClient.execute(scrollGetRequest)
// entity = scrollGetResponse.getEntity()
// def responseMap = objectMapper.readValue(EntityUtils.toString(entity), Map.class)
//
// def hits = responseMap.get("hits").get("hits")
// if (hits.isEmpty()) {
// break // 没有更多数据,跳出循环
// }
//
// // 合并当前页的数据
// allResults.addAll(hits.map({ it.get("_source") }))
//
// // 更新scrollId以进行下一次请求
// scrollId = responseMap.get("_scroll_id").toString()
// }
//
// // 关闭HttpClient
// httpClient.close()
//
// // 输出或处理全量数据
// println(allResults)
//
// // 注意:实际应用中,你可能需要在获取完数据后清除scroll上下文,以释放资源。
// }
}
@@ -0,0 +1,65 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
//更新第三方均价表中小区区域
class RegionUpdateBatch {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static main(args) {
def task = new RegionUpdateBatch()
task.run()
}
def run(){
def citys = CommonDatautils.findAllCity()
for(def city : citys){
def districtMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["city","region", "name"])
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("region", 1)
keys.put("name", 1)
keys.put("seaweed_id", 1)
def statistics = AVERAGE_STATISTICS.find(new BasicDBObject("city", city), keys)
def total = statistics.size()
final CountDownLatch latch = new CountDownLatch(total)
def totalCount = new AtomicLong()
for(def i=0;i<statistics.size();i++){
final def d = statistics[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def dist = districtMap.get(d.seaweed_id)
if(dist && dist.region!=d.region){
def udata = new BasicDBObject()
udata.put("region", dist.region)
AVERAGE_STATISTICS.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata),false,false)
}
println totalCount.get() + " / "+total + " "+ d.city
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
}
}
@@ -0,0 +1,231 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.bson.types.ObjectId
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 第三方小区与我们小区映射关系表
*/
@BatchOwner(Watcher.GaoJuan)
class ThirdPriceMappingBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
final DBCollection thirdPartyPriceColl = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018)
final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018)
final DBCollection dataProcessColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process", "192.168.0.235", 27018)
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
static main(args) {
runBatch(new ThirdPriceMappingBatch(), args);
}
@Override
public void process(Object args) {
// //将新增数据同步过来
// addNemCommunity()
// //接口进行匹配
match()
addOldMatchResult()
}
def match(){
def citys = CommonUtils.findAllCity()
for(def c : citys){
def query = new BasicDBObject("city", c).append("match_id", new BasicDBObject('$exists', false))
def noMatchDatas = thirdSeaweedMappingColl.find(query).toList()
matchByCity(c, noMatchDatas)
}
}
def matchByCity(def c, def noMatchDatas){
def totalCount = new AtomicLong()
def addCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(noMatchDatas.size())
def total = noMatchDatas.size()
for(def i=0;i<noMatchDatas.size();i++){
final def nmd = noMatchDatas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
addCount.addAndGet(1)
def address = nmd.city
if(nmd.district){
address = address + nmd.district
}
address = address + nmd.name
def jsonObject = FangjiaClientUtils.newMatch(nmd.city, address, "property", false)
if(!jsonObject){
if(DataUtil.isPreciseAddress(nmd.address)){
address = nmd.city
if(nmd.district){
address = address + nmd.district
}
address = address + nmd.address
jsonObject = FangjiaClientUtils.newMatch(nmd.city, address, "property", false)
}
}
if(jsonObject) {
def udata = new BasicDBObject()
udata.put("match_id",jsonObject.get("id"))
udata.put("match_city",jsonObject.get("city"))
udata.put("match_district",jsonObject.get("district"))
udata.put("match_name",jsonObject.get("name"))
udata.put("match_type",jsonObject.get("propertyType"))
udata.put("match_credit",jsonObject.get("credit"))
thirdSeaweedMappingColl.update(new BasicDBObject("_id", nmd._id), new BasicDBObject('$set', udata))
println addCount.get()+"/"+total +" [success] "+ udata.get("match_district")+" "+udata.get("match_name")
}else{
println addCount.get()+"/"+total +" [fail] "+address
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return addCount.get();
}
def addNemCommunity(){
def citys = CommonUtils.findAllCity()
def total = 0
def index = 0
for(def c : citys){
index ++
if(index < 70){
continue
}
def crawlDatas = thirdPartyPriceColl.find(new BasicDBObject("city", c).append("s_date", curDate)).toList()
def addCount = addDatasToDB(crawlDatas)
println c + " 新增:"+ addCount
total = total + addCount
}
println total +" 新增总计:"+total
}
def addDatasToDB(def crawlDatas){
def totalCount = new AtomicLong()
def addCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(crawlDatas.size())
for(def i=0;i<crawlDatas.size();i++){
final def sd = crawlDatas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
sd.put("district", sd.region)
sd.removeField("region")
// sd.removeField("url")
sd.removeField("s_date")
totalCount.addAndGet(1)
def query = new BasicDBObject("url", sd.url)
def dbo = thirdSeaweedMappingColl.findOne(query)
if(dbo){
def udata = new BasicDBObject()
sd.each {k,v->
udata.put(k,v)
}
udata.removeField("_id")
udata.put("m_date", new Date())
thirdSeaweedMappingColl.update(query, new BasicDBObject('$set', udata),false, false)
}else{
sd.put("m_date", new Date())
thirdSeaweedMappingColl.save(sd)
addCount.addAndGet(1)
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return addCount.get();
}
def addOldMatchResult(){
def citys = CommonUtils.findAllCity()
def keys = new BasicDBObject()
keys.put("_id", 1)
for(def c : citys){
def datas = thirdSeaweedMappingColl.find(new BasicDBObject("city", c).append("old_match_id", new BasicDBObject('$exists', false)), keys).toList()
println c +" 待关联数据:"+datas.size()
findResultFromDB(c, datas)
println c + " 关联数据结束 "+DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd HH:mm:ss")
}
}
def findResultFromDB(city, datas){
def addCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(datas.size())
def total = datas.size()
for(def i=0;i<datas.size();i++){
final def d = datas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
addCount.addAndGet(1)
def query = new BasicDBObject().append("s_date", curDate).append("city", city).append("tid", d._id.toString())
def obj = dataProcessColl.findOne(query)
if(obj) {
def dist = CommonDatautils.findDistrict(obj.match_city, obj.match_region, obj.match_name, "district")
def udata = new BasicDBObject()
if(dist){
udata.put("old_match_id",dist._id.toString())
}
udata.put("old_match_city",obj.match_city)
udata.put("old_match_district",obj.match_region)
udata.put("old_match_name",obj.match_name)
udata.put("old_match_way",obj.type)
thirdSeaweedMappingColl.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
println addCount.get()+"/"+total +" [旧匹配找到] "
}else{
println addCount.get()+"/"+total +" [旧匹配没找到] "+d._id.toString()
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
}
@@ -0,0 +1,111 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
/**
* 乡镇街道 判断 中心点是否在围栏或一个围栏是否包含多个中心点
*/
@BatchOwner(Watcher.GaoJuan)
class TownCenterBatch extends BaseBatch{
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
static main(args) {
runBatch(new TownCenterBatch(), args)
}
@Override
void process(Object args) {
def city = "上海市"
def shapes = findHasShapeTown(city)
def centers = findAllCenters(city)
def retMap = checkCenter(shapes, centers)
retMap.each {key,values->
if(values.size()>1){
println values
}
}
}
def checkCenter(shapes, centers){
def map = [:]
def index = 0
for(def town : shapes){
index ++
def points = InsidePolygon.createPoint4(town.shape.coordinates)
//1.先判断当前镇的中心点是否在围栏中
if(town.location){
def flag = InsidePolygon.isInArea(town.location.lon, town.location.lat, points)
if(!flag){
println town.city + " "+town.district+" "+town.street+" 不在本身围栏中"
}else{
def list = map.get(town.city + "#"+town.district+"#"+town.street)
if(!list){
list = []
map.put(town.city + "#"+town.district+"#"+town.street, list)
}
list << town.city + "#"+town.district+"#"+town.street
}
}
//2.判断是否有其它点在围栏中
for(def center : centers){
if(center._id.toString() != town._id.toString()){
def flag = InsidePolygon.isInArea(center.location.lon, center.location.lat, points)
if(flag){
def list = map.get(town.city + "#"+town.district+"#"+town.street)
if(!list){
list = []
map.put(town.city + "#"+town.district+"#"+town.street, list)
}
list << center.city + "#"+center.district+"#"+center.street
println center.city + " "+center.district+" "+center.street+" 在 "+town.city + " "+town.district+" "+town.street+" 围栏中"
}
}
}
def list = map.get(town.city + "#"+town.district+"#"+town.street)
if(list){
println index+" "+town.city + "#"+town.district+"#"+town.street + " "+list.size()
}
}
return map
}
def findHasShapeTown(city){
def q = new BasicDBObject()
q.put("type", "乡镇街道")
q.put("shape", new BasicDBObject('$exists', true))
if(city){
q.put("city", city)
}
def keys = new BasicDBObject()
keys.put("location", 1)
keys.put("city", 1)
keys.put("district", 1)
keys.put("street", 1)
keys.put("shape", 1)
return ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).toList()
}
def findAllCenters(city){
def q = new BasicDBObject()
q.put("type", "乡镇街道")
q.put("location", new BasicDBObject('$exists', true))
if(city){
q.put("city", city)
}
def keys = new BasicDBObject()
keys.put("location", 1)
keys.put("city", 1)
keys.put("district", 1)
keys.put("street", 1)
return ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).toList()
}
}
@@ -1,7 +1,11 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.BaiduMapUtils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
@@ -38,6 +42,9 @@ class TownMiddleProcess {
DBCollection REGIONPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
DBCollection TOWNPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
def fullCityMap = CommonDatautils.queryFullNameCity()
static main(args) {
@@ -57,7 +64,7 @@ class TownMiddleProcess {
//修改乡镇街道别名
// updateTownAlias()
//中间表同步到100
synchTown()
// synchTown()
//添加地级市标志
// addCityFlag()
// addPinyin()
@@ -70,7 +77,8 @@ class TownMiddleProcess {
//对比中间表的街道与community街道是否一致,因为元极反馈类似“高密市醴泉街道”解析有问题,要去掉“高密市”,然后去了中间表
// updateCommunityDiffStreet()
// updateTownPoline()
// updateTownPoline()//更新乡镇街道围栏
// updateTownPolineBatch()//批量更新乡镇街道围栏
// checkTown()
//区域民政部名称更新到乡镇街道中
// updateTownshipByRegion()
@@ -83,6 +91,11 @@ class TownMiddleProcess {
// mzbDistrictUpate()
// addCity()
// addTownCenter()//百度接口添加中心点
// checkCenterInTownRang()//检测乡镇街道中心点是否在围栏中
addSpecialCityTownAlias()
}
@@ -258,6 +271,7 @@ class TownMiddleProcess {
def createTown(){
def cat = "乡镇街道"
def tjjMaps = findAllProcincesFromTJJ(cat)
def lastRets = []
def govMap = findAllGovTown()
@@ -853,9 +867,10 @@ class TownMiddleProcess {
}
def updateTownPoline(){
def city = "唐山市"
def name = "高新技术开发区街道"
def q = new BasicDBObject("city", city).append("town","唐山高新技术产业开发区")
def city = "海口市"
def region = "美兰区"
def name = "海口桂林洋经济开发区(农场)"
def q = new BasicDBObject("city", city).append("region", region).append("town","国营桂林洋农场")
println city+" "+ name+" "+TOWNPOLYLINE_COLL.count(q)
def d = TOWNPOLYLINE_COLL.findOne(q)
if(d){
@@ -882,11 +897,102 @@ class TownMiddleProcess {
udata.put("shape", resultMap)
def updateQuery = new BasicDBObject("city", city).append("street",name)
def updateQuery = new BasicDBObject("city", city).append("district", region).append("street",name)
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}
}
def updateTownPolineBatch(){
def i = 0
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\乡镇街道围栏开发区数据.xlsx",{ row->
if(row['R'] && row['R']=="新增围栏"){
i++
if(i>=134){
def q = new BasicDBObject("city", row['B']).append("region", row['C']).append("town",row['D'])
println row['B']+" "+ row['D']+" "+TOWNPOLYLINE_COLL.count(q)
def d = TOWNPOLYLINE_COLL.findOne(q)
if(d){
def resultMap = [:]
def bd_resultMap = [:]
resultMap.type = "polygon"
bd_resultMap.type = "polygon"
def arrs = d.polyline.split("\\|")
def rets = []
def bd_rets = []
println row['N'] +" 共有:"+arrs.size()+" 个围栏"
def index = 1
for(def s : arrs){
def points = s.split(";")
def pointList = []
def bd_pointList = []
for(def pointStr : points){
def pointArr = pointStr.split(",")
def lon = DataHandleUtils.getDoubleRound(6, pointArr[0].toString().toDouble())
def lat = DataHandleUtils.getDoubleRound(6, pointArr[1].toString().toDouble())
pointList << [lon, lat]
// 高德转百度
def baidu = BaiDuApiUtils.geoconv(lon, lat, 3)
def bd_lng = baidu.x
def bd_lat = baidu.y
bd_pointList << [bd_lng, bd_lat]
}
if(index<=1){
resultMap.coordinates = pointList
bd_resultMap.coordinates = bd_pointList
def udata = new BasicDBObject()
udata.put("shape", resultMap)
udata.put("operator", "高娟")
udata.put("m_date", new Date())
udata.put("bd_shape",bd_resultMap)
def updateQuery = new BasicDBObject("name",row['N'])
DEVELOPMENT_ZONE_43_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}else{
def dbo = DEVELOPMENT_ZONE_43_COLL.findOne(new BasicDBObject("name",row['N']))
def obj = new BasicDBObject()
dbo.each {k,v->
if(k!='_id'){
obj.put(k,v)
}
}
resultMap.coordinates = pointList
bd_resultMap.coordinates = bd_pointList
obj.put("shape", resultMap)
obj.put("operator", "高娟")
obj.put("m_date", new Date())
obj.put("bd_shape",bd_resultMap)
obj.put("create_time", new Date())
obj.put("block", dbo.name+"-地块"+index)
DEVELOPMENT_ZONE_43_COLL.save(obj)
}
// rets << pointList
// bd_rets << bd_pointList
index++
}
// if(rets.size()>1) {
// resultMap.type = "MultiPolygon"
// bd_resultMap.type = "MultiPolygon"
// }
// println d.city+" "+d.region+" "+d.town+" "+rets.size()
row['S'] = "已完成"
}
}
}
})
}
def checkTown(){
def q = new BasicDBObject("shape", new BasicDBObject('$exists', false)).append("type","乡镇街道")
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
@@ -1164,4 +1270,90 @@ class TownMiddleProcess {
def updateQuery = new BasicDBObject("_id", id)
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}
def addTownCenter(){
ExcelUtils.modify("D:\\数据\\数据建设\\乡镇街道\\乡镇街道无中心点数据.xlsx",{ row->
def json = BaiduMapUtils.searchByAddress(row['C']+row['D']+row['E'])
if(json){
row['J'] = json.confidence
row['K'] = json.comprehension
row['L'] = json.level
def location = json.location;
def gmap = GeocodeUtils.geoconvBaiduToGoogle(location.lng.toDouble(), location.lat.toDouble())
if(gmap){
gmap.lng = DataHandleUtils.getDoubleRound(6, gmap.lng)
gmap.lat = DataHandleUtils.getDoubleRound(6, gmap.lat)
row['F'] = gmap.lng
row['G'] = gmap.lat
row['H'] = location.lng.toDouble()
row['I'] = location.lat.toDouble()
}
}
})
}
def checkCenterInTownRang(){
def q = new BasicDBObject()
q.put("type","乡镇街道")
q.put("location", new BasicDBObject('$exists',false))
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
for(def d : dbcursor){
}
}
def addSpecialCityTownAlias(){
def fullCityMap = CommonDatautils.queryFullNameCity()
def citys = ["东莞","中山","嘉峪关","昆玉","石河子","阿拉尔","图木舒克","五家渠","济源","仙桃","潜江","天门","神农架","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中","北屯","铁门关","双河","可克达拉"]
def regionMap = CommonDatautils.getAllCityFullRegions()
for(def city : citys){
def townMap = regionMap.get(city)
def streets = findAllStreetByCity(city)
for(def s : streets){
def set = townMap.keySet()
def alias = null
for(def name : set){
def townObj = townMap.get(name)
if(townObj && townObj.pattern && townObj.pattern.contains(s.street)){
alias = townObj.pattern
break;
}
}
if(alias){
def newAlias = []
for(def n : alias){
if(n!=s.street){
newAlias << n
}
}
if(alias){
def udata = new BasicDBObject()
udata.put("alias", newAlias)
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id", s._id), new BasicDBObject('$set', udata),false, false)
}
}
}
}
}
def findAllStreetByCity(city){
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
fullCityName = data.city
}
def q = new BasicDBObject()
q.put("type", "乡镇街道")
q.put("city", fullCityName)
def keys = new BasicDBObject()
keys.put("street", 1)
return ADMINISTRATIVE_MAPPING_COLL.find(q, keys).toList()
}
}
@@ -0,0 +1,319 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import org.apache.commons.io.FileUtils
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.update.UpdateRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
/**
* 区域标准化
* 1.小区库乡镇街道标准化,包含小区,商铺,写字楼
* 2.poi数据库中乡镇街道标准化
*/
class TownStandardBatch extends BaseBatch{
def fullCityMap = CommonDatautils.queryFullNameCity()
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
def updateList = []
static main(args) {
runBatch(new TownStandardBatch(), args)
}
@Override
void process(Object args) {
TownStandard()
}
def TownStandard(){
// townStandardByCat("office")
townStandardByPoi()
}
def townStandardByCat(def cat){
//"三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德",
def citys = ["重庆","龙岩","南平","漳州","福州","厦门","泉州","莆田","三明","宁德"]//CommonDatautils.findAllCity()
for(def city : citys){
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
fullCityName = data.city
}
def mzbRegionMap = CommonDatautils.queryMzbRegion(fullCityName)
def shortRegionNameMap = CommonDatautils.getRegionShortName(fullCityName)
def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district", "street", "alias"], mzbRegionMap)
if(townMap){
def regionSet = townMap.keySet()
if(regionSet.size()>1){
haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap)
}else{
println city +" 无区域"
}
}
}
}
def townStandardByPoi(){
updateList << ["数据ID", "城市","区域","乡镇街道","名称","状态", "修改"]
def citys = CommonDatautils.findAllCity()
for(def city : citys){
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
fullCityName = data.city
}
if(fullCityName){
def fileName = "/home/goojia/gj/poi/${fullCityName}.csv"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\${fullCityName}.csv"
}
File file=new File(fileName);
if(file.exists()){
def rMap = regionFullNameMap.get(fullCityName)
if(rMap){
def mzbRegionMap = CommonDatautils.queryMzbRegion(fullCityName)
def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district","mzb_district_name", "street", "alias"])
def set = rMap.keySet()
// def rets = findEsPoiByCity(fullCityName)
def rets = readPoiCsv(fullCityName)
println city +" "+rets.size()
if(set.size()>1){
poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap)
}else{
// haveStreetProcess(city, set, cat)
}
}
}
}else{
println "没找到相应的全称:"+city
}
}
ExcelUtils.write("D:\\数据\\乡镇街道待查数据.xlsx", updateList, "数据")
}
def haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap){
def results = []
if(fullCityName){
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name", "township"])
for(def d : districts){
if(d.township){
def towns = null
def updateTown = null
for(def m : townMap){
def fullRegionName = shortRegionNameMap.get(d.region)
if(m.key == fullRegionName){
towns = m.value
break
}
}
def opt = true
if(towns){
for(def town : towns){
def townAlias = []
if(town.alias){
townAlias = town.alias
}
townAlias << town.street
if(townAlias.contains(d.township)){
if(d.township!=town.street){
updateTown = town.street
}else{
opt = false
}
break
}
}
}
if(opt){
if(updateTown){
def udata = new BasicDBObject()
udata.put("township", updateTown)
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name+" "+d.township +" 乡镇修改为:"+updateTown
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
}else{
def udata = new BasicDBObject()
udata.put("township", "")
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name+" "+d.township +" 乡镇修改为:空值"
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
}
}
}
}
}
}
def poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap){
def results = []
if(fullCityName){
for(def d : rets){
if(d.township){
def towns = null
def updateTown = null
for(def m : townMap){
def townDatas = townMap.get(d.district)
if(townDatas){
towns = townDatas
break
}else{
println(d.city +" "+d.district+" 没找下辖街道")
}
}
def opt = true
if(towns){
for(def town : towns){
def townAlias = []
if(town.alias){
townAlias = town.alias
}
townAlias << town.street
if(townAlias.contains(d.township)){
if(d.township!=town.street){
updateTown = town.street
}else{
opt = false
}
break
}
}
}
if(opt){
if(updateTown){
def m = [:]
m.put(d.id, ["township":updateTown])
results << m
// if(results.size()%100==0){
// updateByEs(results)
// results.clear()
// }
updateList << [d.id, d.city, d.district, d.name,d.township, d.status, updateTown]
println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown
}else{
def m = [:]
m.put(d.id, ["township":""])
results << m
// if(results.size()%100==0){
// updateByEs(results)
// results.clear()
// }
updateList << [d.id, d.city, d.district, d.name, d.township, d.status, ""]
println d.city +" "+d.district+" "+d.name +" "+d.township+" 状态修改为:空值"
}
}
}
}
}
// if(results.size()>0){
// updateByEs(results)
// results.clear()
// }
println fullCityName +" 处理完成"
}
def updateByEs(rows){
def client = ElasticUtils89.getClient("192.168.0.89")
def bulkRequest = new BulkRequest()
rows.each { row ->
def id = row.keySet().iterator().next()
def doc = row.get(id)
def updateRequest = new UpdateRequest("didata_map_poi", id)
updateRequest.doc(doc, XContentType.JSON)
bulkRequest.add(updateRequest)
}
try {
// 执行批量更新请求
def bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT)
// 检查响应
if (bulkResponse.hasFailures()) {
println("批量更新中有失败的操作:")
bulkResponse.buildFailureMessage().each { println(it) }
} else {
println("批量更新成功!")
}
} catch (Exception e) {
e.printStackTrace()
} finally {
// 关闭客户端
try {
client.close()
} catch (Exception e) {
e.printStackTrace()
}
}
}
def findEsPoiByCity(city){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("status", 0))
def rets = []
String[] includes = ["city", "district", "name", "township", "status", "id"]
def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi", queryBuilder, includes)
for(def d: datas){
def o = new BasicDBObject()
o.put("city", d.city)
o.put("district", d.district)
o.put("name", d.name)
o.put("township", d.township)
o.put("status", d.status)
o.put("id", d.id)
rets << o
}
return rets
}
def readPoiCsv(filename){
def datas = readCSV(filename)
return datas
}
def readCSV(filename){
def lines = FileUtils.readLines(new File(filename))
def rets = [];
for(line in lines) {
if(line){
String[] arrs = line.split("\\|@\\|")
def o = new BasicDBObject()
o.put("city", arrs[0])
o.put("district", arrs[1])
o.put("name", arrs[2])
o.put("township", arrs[3])
o.put("status", arrs[4])
o.put("id", arrs[5])
rets << o
}
}
return rets;
}
}
@@ -42,7 +42,7 @@ class TownshipStrandUpdate {
// task.updateTown()
// task.stat()
// task.updateDiffRegionTown()
task.updateDiffRegionTown()
// task.cleanTown()
// task.checkTownFromMongo()
// task.updateTownMapping()
@@ -423,29 +423,29 @@ class TownshipStrandUpdate {
}
def updateDiffRegionTown(){
def fileName = "D:\\数据\\数据建设\\乡镇街道\\shop-不同区域.xlsx"
ExcelUtils.multiThreadingModify(fileName,{d->
def lists = CommonDatautils.findDistrictList(d['B'], null, d['D'], "shop")
def fileName = "D:\\数据\\数据建设\\乡镇街道\\district-不同区域.xlsx"
ExcelUtils.modify(fileName,{d->
def lists = CommonDatautils.findDistrictList(d['A'], null, d['C'], "district")
d['AA'] = lists.size()
//只处理库中只有一个的
if(lists && lists.size()==1 && d['Z']=="包含" && !d['AB']){
def newRegion = RegionData.formatRegion(d['B'], d['M'])
if(lists && lists.size()==1 && d['R']){
def newRegion = RegionData.formatRegion(d['A'], d['R'])
if(newRegion){
if(d['X'] && d['X'].toString().contains(d['M']+" "+d['N'])){ //地图街道在中间表中
// if(d['X'] && d['X'].toString().contains(d['M']+" "+d['N'])){ //地图街道在中间表中
def dbo = lists[0]
def arrs = d['X'].split(" ")
// def arrs = d['X'].split(" ")
def udata = new BasicDBObject()
udata.put("region", newRegion)
udata.put("m_date", new Date())
udata.put("township", arrs[1])
// udata.put("township", arrs[1])
DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata))
CommonDatautils.saveOptLog(dbo.city, dbo.region, dbo.name, "shop", "update","高娟")
CommonDatautils.saveOptLog(dbo.city, dbo.region, dbo.name, "district", "update-region","高娟")
d['AB'] = "已处理"
}
// }
}
}
})
},1)
}
def cleanTown(){
@@ -47,7 +47,7 @@ class DistrictFieldAddBatch extends BaseBatch{
//"alias":"alias",
//ownership_type:权属类型 为商品房、经适房、动迁房等
//house_property 产权性质:国有产权 "house_property":"网站没有",
def fieldsMap = ["estate_charge":"property_fee","block":"block","build_date":"build_year","estate_company":"property_company","developer":"developer",
def fieldsMap = ["lng":"location","estate_charge":"property_fee","block":"block","build_date":"build_year","estate_company":"property_company","developer":"developer",
"virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area","household_count":"households",
"car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address","build_structure":"building_structure",
"build_type":"build_type", "land_property":"ownership_type","live_date":"live_date",
@@ -115,7 +115,7 @@ class DistrictFieldAddBatch extends BaseBatch{
println "【通过匹配填充字段】网站:"+seaweedmapSource +" Es:"+esRets.size()+" 融合小区占比:"+DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/totalCount)+"%"
def precentMap = fieldUpdate(totalCount, updateRets)
println c +" "+precentMap.toString()
//updateDB(c, updateRets)
updateDB(c, updateRets)
println "------------------------------------------------------------------------------"
def pre1 = 0
def pre2 = 0
@@ -212,6 +212,11 @@ class DistrictFieldAddBatch extends BaseBatch{
def rets = []
rets << ["第三方城市","第三方区域","第三方名称","第三方地址","第三方户数","匹配区域","匹配名称","匹配可信度","方式"]
for(def matchDist : matchRets){
// if(!["智富中心","龙山新寓","天怡瀚园","远盛凤林府","京口路小区","梦溪路小区","浩创优尚","溪山丽苑","旭园"].contains(matchDist.name)){
// continue
// }
def dist = CommonDatautils.findDistrictFieldById(matchDist.match_id, fields)
if(!dist){
println c+" "+matchDist.match_id+" 不存在"
@@ -306,6 +311,15 @@ class DistrictFieldAddBatch extends BaseBatch{
if(newVal){
newValues.put(seaweedfield, newVal)
}
if(seaweedfield == "lng"){
def locationMap = FiledStrand.processLngAndLat(esVal)
if(locationMap){
newValues.put("lat", locationMap.lat)
newValues.put("lng", locationMap.lng)
newValues.put("lat2", locationMap.lat2)
newValues.put("lng2", locationMap.lng2)
}
}
}
}
}
@@ -248,10 +248,10 @@ class FangjiaClientUtils {
def content = null
def url = null;
try {
address = address.replace("{","").replace("\\","")
// url = "http://192.168.0.93:8080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
// address = address.replace("{","").replace("\\","")
url = "http://192.168.0.93:8080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
// url = "http://192.168.20.242:5001/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
// url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
if(city){
url = url + "&city="+URLEncoder.encode(city,"UTF-8")
}
@@ -0,0 +1,56 @@
package com.fangjia.script.gj.district
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
/**
* 临时移除 相同名称即在正常状态下有,在删除状态下也有的小区
*/
class RemoveDistrict {
public static void main(String[] args) {
def task = new RemoveDistrict()
task.run()
}
def run(){
def citys = CommonUtils.findAllCity()
def cat = "shop"
def keys = []
citys.each {c->
def districts = CommonDatautils.findAllDistricts(c, cat, ["city", "region", "name", "status"])
districts.each {it->
def key = it.city +"##"+it.region+"##"+it.name
if(!keys.contains(key)){
keys << key
}else{
println it.toString()+" 删除"
DbUtils.seaweed.remove(new BasicDBObject("_id",it._id))
}
// def list = CommonDatautils.findAllStatusDistrict(it.city, it.region, it.name, cat, ["city", "region", "name", "status"])
// if(list && list.size()>1){
// println it.toString() + " "+list.size()
// for(def d : list){
//// if(d.status.toString()=="0"){
//// println d.toString()+" 保留"
//// }else if(d.status.toString()=="1"){
//// println d.toString()+" 删除"
//// DbUtils.seaweed.remove(new BasicDBObject("_id",d._id))
//// }
//
// if(d.status.toString()=="1"){
// println d.toString()+" 保留"
// }else if(d.status.toString()!="1"){
// println d.toString()+" 删除"
// DbUtils.seaweed.remove(new BasicDBObject("_id",d._id))
// }
// }
//
// println "................................................................................................"
// }
}
}
}
}
@@ -34,7 +34,7 @@ class FjwBigdataCommbuildtypeBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def citys = CcbCommonUtils.getRunningCitys(["all"])
def rets = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
@@ -43,7 +43,7 @@ class FjwBigdataCommunityBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def citys = CcbCommonUtils.getRunningCitys(["all"])
def rows = []
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
@@ -50,7 +50,7 @@ class FjwBigdataDealBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def citys = CcbCommonUtils.getRunningCitys(["all"])
// println "curDate:"+curDate
// def allCitys = CommonDatautils.findAllCity()
@@ -85,7 +85,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{
public void process(Object args) {
//目前查询库中是否已存在,用的是Oracle
def index = 0
def citys = CcbCommonUtils.getRunningCitys(args)
def citys = CcbCommonUtils.getRunningCitys(["all"])
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
@@ -44,7 +44,7 @@ class FjwGuiDingPriceBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def citys = CcbCommonUtils.getRunningCitys(["all"])
def rows = []
FjwPgsqlCommUtils.deleteGuidedPrice(curDate)
@@ -46,7 +46,7 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def citys = CcbCommonUtils.getRunningCitys(["all"])
runDatas(citys)
//同步历史数据
@@ -0,0 +1,146 @@
package com.fangjia.script.gj.images
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 更新小区图片
*/
@BatchOwner(Watcher.GaoJuan)
class AddDistImagesBatch extends BaseBatch{
final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
public static final DBCollection seaweed_fs_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_images", "mongo.db.fangjia.com")
static main(args) {
runBatch(new AddDistImagesBatch(), args);
}
@Override
public void process(Object args) {
// def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
def mdate = DateUtils.getFirstDayOfMonth(new Date(), -1)
def citys = CommonUtils.findAllCity()
citys.each {city->
def communitys = findAllEsData2(city, mdate)
println "抓取城市: "+city +" 第三方小区有图片数:"+communitys.size()
//第三方匹配小区
matchCommunitys(communitys)
}
}
def findAllEsData2(city, mdate){
QueryBuilder date_range = QueryBuilders.rangeQuery("m_date").from(mdate.getTime()).includeLower(true)
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("source", "安居客小程序"))
.must(date_range).must(date_range)
def rets = []
def datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "community", queryBuilder);
if(datas) {
for (int k = 0; k < datas.size(); k++) {
def d = datas[k];
if(d.house_type_images_2 || d.outdoor_images) {
def data = new BasicDBObject()
data.put("city", d.city)
data.put("region", d.region)
data.put("name", d.name)
data.put("id", d.id)
data.put("out_images", d.outdoor_images)
data.put("house_type_images", d.house_type_images_2)
rets << data
}
}
}
return rets;
}
def findAllEsData(city, mdate){
def names = new HashSet()
def rets = []
ElasticUtils.executeSqlQuery("select * from community where source='安居客小程序' and city='${city}' and m_date>='${mdate}' order by m_date", { d->
if(d.house_type_images_2 || d.outdoor_images){
def s = d.city +"#"+d.region+"#"+d.name+"#"+d.property_type
if(!names.contains(s)){
names << s
def data = new BasicDBObject()
data.put("city", d.city)
data.put("region", d.region)
data.put("name", d.name)
data.put("out_images", d.outdoor_images)
data.put("house_type_images", d.house_type_images_2)
rets << data
}
}
})
return rets;
}
def matchCommunitys(communitys){
def addCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(communitys.size())
def rets = new CopyOnWriteArrayList()
def total = communitys.size()
for(def i=0;i<communitys.size();i++){
final def d = communitys[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
try{
addCount.addAndGet(1)
def query = new BasicDBObject("city", d.city).append("district",d.region).append("name",d.name).append("source","安居客").append("match_id", new BasicDBObject('$exists', true))
def obj = thirdSeaweedMappingColl.findOne(query)
if(obj) {
def vo = new BasicDBObject()
vo.put("city", obj.match_city)
vo.put("district", obj.match_district)
vo.put("name", obj.match_name)
if(d.out_images){
vo.put("out_pics", d.out_images)
}
if(d.house_type_images){
vo.put("floorplan_pics", d.house_type_images)
}
vo.put("m_date", new Date())
rets.add(vo)
def q = new BasicDBObject("_id", new ObjectId(obj.match_id))
seaweed_fs_coll.update(q, new BasicDBObject('$set', vo), true,false)
println addCount.get()+"/"+total +" [匹配库中存在] "
}else{
println addCount.get()+"/"+total +" [没找到相应的匹配数据] "+d._id.toString()
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
// }
// });
}
latch.await()
return rets
}
}
@@ -2,6 +2,7 @@ package com.fangjia.script.gj.poc
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
@@ -25,6 +26,10 @@ class AddPriceTest {
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver")
//235为房价网内部计算均价的库
def pgsqlInstance235 = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
public static final DBCollection confirm_price_date = MongoUtils.getCollectionByName("fangjia", "confirm_price_data", "mongo.db.fangjia.com")
@@ -33,9 +38,9 @@ class AddPriceTest {
task.run();
}
def run(){
def filePath = "D:\\数据\\建行\\房产估值\\20240716-20240815 _业务反馈价格.xlsx"
// addGXDPrice(filePath)
addCcbPrice(filePath)
def filePath = "D:\\数据\\建行\\房产估值\\202410测试偏差较大案例统计数据test.xlsx"
addGXDPrice(filePath)
// addCcbPrice(filePath)
// addThirdPrice(filePath)
// addFjwPrice(filePath)
// exportPkPrice()
@@ -173,17 +178,22 @@ class AddPriceTest {
def cityId = cityCodeDataMap.get(d['V'])
def map = findAvgPrice(d['U'], cityId)
if(map){
d['F'] = map.avg_price
// d['P'] = map.calc_type
d['H'] = map.avg_price
d['I'] = map.calc_type
// d['AM'] = map.credit_value
}
// def newPriceMap = findNewFjwAvgPrice(d['U'], cityId)
// if(newPriceMap){
// d['Q'] = newPriceMap.avg_price
// }
}
})
}
def findAvgPrice(def commid, def cityid){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202407 order by gujiabds_clc_yrmo_val desc limit 1"
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202409 order by gujiabds_clc_yrmo_val desc limit 1"
def map = [:]
pgsqlInstance.eachRow(sql, { r ->
map.avg_price = r.gujia_bds_clc_avprc
@@ -202,6 +212,25 @@ class AddPriceTest {
}
def findNewFjwAvgPrice(def commid, def cityid){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from price_stat.avm_community_base_price where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202409 order by gujiabds_clc_yrmo_val desc limit 1"
def map = [:]
pgsqlInstance235.eachRow(sql, { r ->
map.avg_price = r.gujia_bds_clc_avprc
map.type = r.gujia_bds_clmtd
map.month = r.gujiabds_clc_yrmo_val
map.credit_value = r.gujia_bds_cfdc_dgr
if(r.gujia_bds_clmtd == 1){
map.calc_type = "人工估值"
}else if(r.gujia_bds_clmtd == 2){
map.calc_type = "样本计算"
}else if(r.gujia_bds_clmtd == 3){
map.calc_type = "神经网络"
}
})
return map
}
def addThirdPrice(filepath){
ExcelUtils.multiThreadingModify(filepath, { d ->
if (!d['X']) {
@@ -241,12 +270,16 @@ class AddPriceTest {
def addGXDPrice(filepath){
ExcelUtils.multiThreadingModify(filepath, { d ->
if (!d['M']) {
def dist = CommonDatautils.findDistrictById(d['B'])
if (!d['B']) {
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
// def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
d['M'] = gxdPrice
if(gxdPrice){
d['B'] = gxdPrice.price
d['C'] = DateUtils.formatDateByPattern(gxdPrice.c_date, "yyyy-MM-dd")
}
}
}
})
@@ -260,11 +293,11 @@ class AddPriceTest {
q.put("district_name", name)
q.put("source", "GXD")
def dbcursor = confirm_price_date.find(q).sort(new BasicDBObject("c_date", -1)).limit(1)
def price = null
def obj = null
for(def d : dbcursor){
price = d.price
obj = d
}
return price
return obj
}
@@ -27,7 +27,7 @@ class QiliangTest {
// task.run()
def data = [
'address': '浙江省湖州市长兴县煤山镇工业园区山鹰大道7号',
'address': '浙江省嘉兴市南湖区昌盛南路36号19幢嘉兴智慧产业创业园智慧大厦A座168室',
'type': '1' //1:对公 0:对私
]
def s = task.getProxyResult(data);
@@ -26,6 +26,7 @@ import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.search.Scroll
import org.elasticsearch.search.SearchHit
import org.elasticsearch.search.builder.SearchSourceBuilder
import org.elasticsearch.search.fetch.subphase.FetchSourceContext
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
@@ -35,7 +36,6 @@ import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.function.BiConsumer
/**
* ¹¦ÄÜ˵Ã÷:Elasticsearch¹¤¾ßÀà
*/
@@ -228,14 +228,21 @@ class ElasticUtils89 {
}
static def executeSimpleQuery(String host, String indexName, QueryBuilder qb){
static def executeSimpleQuery(String host, String indexName, QueryBuilder qb, includes){
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
RestHighLevelClient client = getClient(host)
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.scroll(scroll);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
if(includes){
String[] excludes = [] // 没有要排除的字段
FetchSourceContext fetchSourceContext = new FetchSourceContext(true, includes, excludes)
searchSourceBuilder.fetchSource(fetchSourceContext)
}
searchSourceBuilder.query(qb);
// searchSourceBuilder.size(1000)
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
String scrollId = searchResponse.getScrollId();
@@ -89,7 +89,7 @@ class GaoDePoiDataProcess extends BaseBatch{
// synchNetwork()
// println DateUtils.parseFromTightByPattern("2023-11-30","yyyy-MM-dd").getTime()
// sysnchPoi()
sysnchVillage()
// sysnchVillage()
// addTownship()
}
@@ -0,0 +1,184 @@
package com.fangjia.script.gj.xiamen
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
/**
* 厦门涉台园区数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class RelatedTaiWanBatch extends BaseBatch{
def curDate = Integer.parseInt(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"));
DBCollection yuanqu_qiye_coll = MongoUtils.getCollectionByName("crawl", "qzd_shetai_company_detail", "192.168.0.206", 27020)
def relateTaiwn_yuanqu_datas = ["福州台商投资区","杏林台商投资区","集美台商投资区","海沧台商投资区","漳州台商投资区","泉州台商投资区","漳浦台湾农民创业园","漳平台湾农民创业园","仙游台湾农民创业园","清流台湾农民创业园","福清台湾农民创业园","惠安台湾农民创业园","海峡两岸集成电路产业合作试验区(厦门、泉州产业园)","海峡两岸生技和医疗健康产业合作区","海峡两岸农业合作试验区","海峡两岸(三明)现代林业合作实验区","漳州古雷石化基地","永泰闽台农业融合发展产业园","上杭闽台农业融合发展产业园","同安闽台农业融合发展产业园","南靖闽台农业融合发展产业园","漳浦闽台农业融合发展产业园","永春闽台农业融合发展产业园","永安闽台农业融合发展产业园","邵武闽台农业融合发展产业园","平潭闽台农业融合发展产业园","南靖闽台精密机械产业园"]
DBCollection XM_RELATED_TW_QIYE_COLL = MongoUtils.getCollectionByName("business", "xm_related_tw_qiye", "mongo.db.fangjia.com")
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
def zimaoqu_datas = ["中国(福建)自由贸易试验区厦门片区","中国(福建)自由贸易试验区福州片区","中国(福建)自由贸易试验区平潭片区"]
static main(args) {
runBatch(new RelatedTaiWanBatch(), args);
}
@Override
public void process(Object args) {
def day = new Date().getDate()
if(day == 10){
//createDatas() //企业库获取涉台园区企业
updateRangeByPoint() //更改企业所在自贸区
export() //导出数据
}else{
exportEmptyFile()
}
}
def updateRangeByPoint(){
def zimaoqu_range_datas = []
for(def s : zimaoqu_datas){
def dbcursor = DEVELOPMENT_ZONE_43_COLL.find(new BasicDBObject("name", s))
for(def obj : dbcursor){
if(obj && obj.shape){
zimaoqu_range_datas << ["name":obj.name, "block":obj.block,"shape":obj.shape.coordinates]
}
}
}
def dbcursor = XM_RELATED_TW_QIYE_COLL.find(new BasicDBObject())
println dbcursor.count()
for(def r : dbcursor){
def resKey = null
if(r.lon && r.lat){
for(def m : zimaoqu_range_datas){
// def flag = InsidePolygon.isInArea(r.lon, r.lat, InsidePolygon.createPoint5(m.shape))
def flag = InsidePolygon.isPtInPoly(r.lon, r.lat, InsidePolygon.createPoint5(m.shape))
if(flag){
resKey = m.name
break
}
}
}
if(resKey){
def udata = new BasicDBObject("free_trade_zone", resKey)
def updateQuery = new BasicDBObject("_id",r._id)
XM_RELATED_TW_QIYE_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}
}
}
def createDatas(){
def datas = findQiyeDatas()
for(def r : datas){
def map = getLocationInfo(r.city, r.address+r.company)
if(map && map.location){
def arrs = map.location.split(",")
r.put("lon", DataHandleUtils.getDoubleRound(6, arrs[0].toString().toDouble()))
r.put("lat", DataHandleUtils.getDoubleRound(6, arrs[1].toString().toDouble()))
r.put("level", map.level)
}
saveToUpdate(r)
}
}
def saveToUpdate(datas){
def udata = new BasicDBObject()
datas.each{k,v->
udata.put(k,v)
}
XM_RELATED_TW_QIYE_COLL.save(udata)
// def updateQuery = new BasicDBObject("name",row['N'])
// XM_RELATED_TW_QIYE_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}
def findQiyeDatas(){
def datas = []
relateTaiwn_yuanqu_datas.each {
def q = new BasicDBObject("name", it)
def dbcursor = yuanqu_qiye_coll.find(q)
for(def d : dbcursor){
datas << ["_id":d._id, "province":d.province, "city":d.city,"address":d.address, "company":d.company, "name":d.name]
}
println it +" "+ datas.size()
}
return datas
}
def getLocationInfo(def city, def address){
def gloJson = GaoDeApiUtils.geo(city, address)
def map = [:]
if(gloJson) {
def glocation = null
def level = null
if (gloJson && gloJson.get("geocodes")) {
def geocodes = gloJson.get("geocodes")
glocation = geocodes.get(0).location
level = geocodes.get(0).level
}
map.location = glocation
map.level = level
return map
}
return null
}
def export(){
def rows = []
def dbcursor = XM_RELATED_TW_QIYE_COLL.find(new BasicDBObject())
for(def r : dbcursor){
rows << [r._id.toString(), r.province, r.city, r.address, r.company, r.name, r.free_trade_zone]
}
def fileName = "/home/goojia/xmbank/free_trade_zone_${curDate}_init.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\厦门\\涉台园区\\free_trade_zone_${curDate}_init.dat"
}
def allDatas = []
allDatas.addAll(rows)
// FileUtils.batchCreateFile(rows, fileName, null, "utf-8", false);
FileUtils.batchCreateFileByCustomSuffix(rows, fileName, null, "utf-8", false, "|@|","|+|");
XmBankFtpUtils.ftpUpload(fileName, curDate)
def fileName2 = "/home/goojia/xmbank/free_trade_zone_${curDate}_init.dat.rdy"
if(CommonDatautils.isWindow()){
fileName2 = "D:\\数据\\厦门\\涉台园区\\free_trade_zone_${curDate}_init.dat.rdy"
}
def rdy_datas = [rows.size().toString()]
FileUtils.batchCreateFileByCustomSuffix([rdy_datas], fileName2, null, "utf-8", false, "","|+|");
XmBankFtpUtils.ftpUpload(fileName2, curDate)
}
def exportEmptyFile(){
def fileName = "/home/goojia/xmbank/free_trade_zone_${curDate}_init.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\厦门\\涉台园区\\free_trade_zone_${curDate}_init.dat"
}
FileUtils.batchCreateFileByCustomSuffix([], fileName, null, "utf-8", false, "|@|","|+|");
XmBankFtpUtils.ftpUpload(fileName, curDate)
def fileName2 = "/home/goojia/xmbank/free_trade_zone_${curDate}_init.dat.rdy"
if(CommonDatautils.isWindow()){
fileName2 = "D:\\数据\\厦门\\涉台园区\\free_trade_zone_${curDate}_init.dat.rdy"
}
def rdy_datas = ["0"]
FileUtils.batchCreateFileByCustomSuffix([rdy_datas], fileName2, null, "utf-8", false, "","|+|");
XmBankFtpUtils.ftpUpload(fileName2, curDate)
}
}
@@ -262,8 +262,8 @@ class SeaweedDataSynch extends BaseBatch{
map.finish_build_time = dist.create_year?dist.create_year.toString():null
if(dist.lng){
def loc = [:]
loc.lon = DataHandleUtils.getDoubleRound(8, dist.lng/10000000000.0)
loc.lat = DataHandleUtils.getDoubleRound(8, dist.lat/10000000000.0)
loc.lon = DataHandleUtils.getDoubleRound(6, dist.lng/10000000000.0)
loc.lat = DataHandleUtils.getDoubleRound(6, dist.lat/10000000000.0)
map.location = loc
}
@@ -272,8 +272,8 @@ class SeaweedDataSynch extends BaseBatch{
def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon)
if(baiduMap){
def location = [:]
location.lon = DataHandleUtils.getDoubleRound(8, baiduMap.lng)
location.lat = DataHandleUtils.getDoubleRound(8, baiduMap.lat)
location.lon = DataHandleUtils.getDoubleRound(6, baiduMap.lng)
location.lat = DataHandleUtils.getDoubleRound(6, baiduMap.lat)
map.bd_location = location
}
@@ -281,8 +281,8 @@ class SeaweedDataSynch extends BaseBatch{
def gpsLoactionMap = GeoUtils.gcjToGps84(map.location.lat, map.location.lon)
if(gpsLoactionMap){
def gpslocation = [:]
gpslocation.lon = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lng)
gpslocation.lat = DataHandleUtils.getDoubleRound(8, gpsLoactionMap.lat)
gpslocation.lon = DataHandleUtils.getDoubleRound(6, gpsLoactionMap.lng)
gpslocation.lat = DataHandleUtils.getDoubleRound(6, gpsLoactionMap.lat)
map.gps_location = gpslocation
}
}
@@ -0,0 +1,76 @@
package com.fangjia.script.gj.xiamen
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.jiangnanblank.SFTP
import com.jcraft.jsch.ChannelSftp
import com.jcraft.jsch.SftpATTRS
import org.apache.commons.codec.digest.DigestUtils
class XmBankFtpUtils {
static def ftpServer = "192.168.0.86"
static def ftpPort = 17495
static def ftpUser = "xmbank"
static def ftpPassword = "Xmbank123!@#"
static def ftpPath = "/output/free_trade_zone"
static SFTP sftp = new SFTP();
static ChannelSftp sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword);
static def ftpUpload(outputPath_dist, month){
//上传到ftp
sftpChannel = reConnect();
sftp.upload(ftpPath, outputPath_dist, sftpChannel);
}
static def reConnect(){
if(sftpChannel == null){
println "sftpChannel 为空,需重连....."
if(sftp == null){
sftp = new SFTP();
}
sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword);
}else{
println "sftpChannel 正常....."
}
return sftpChannel;
}
static def uploadSha512Data(def sb, def dataCodeFileName, def charset){
String sha512HexStr= DigestUtils.sha512Hex(sb.getBytes())//生成加密码码
DataHandleUtils.outPutFile(dataCodeFileName, sha512HexStr,charset);//生成加密码文件
ftpUpload(dataCodeFileName)//上传加密码文件到FTP
}
static def uploadSha512Data2(def dataFileName, def dataCodeFileName, def charset){
def data = new FileInputStream(new File(dataFileName))
String sha512HexStr= DigestUtils.sha512Hex(data)//生成加密码码
DataHandleUtils.outPutFile(dataCodeFileName, sha512HexStr,charset);//生成加密码文件
ftpUpload(dataCodeFileName)//上传加密码文件到FTP
}
static def listFileNames(month){
//上传到ftp
sftpChannel = reConnect();
Vector files = sftp.listFiles(ftpPath+ month+"/", sftpChannel);
return files
}
static def getFileLength(String path, String month){
sftpChannel = reConnect();
def size = sftp.getFileSize(ftpPath+ month+"/"+path, sftpChannel)
return size
}
static main(args) {
}
}
@@ -115,6 +115,20 @@ class InsidePolygon {
}
return alllist
}
public static List<Point> createPoint5(def pointList){
List<Point> alllist = new ArrayList<>();
for(def pointArr : pointList){
Point point = new Point();
double x = pointArr[0];
double y = pointArr[1];
point.setLon(x);
point.setLat(y);
alllist.add(point);
}
return alllist
}
/**
* 判定一个经纬度点与区域的关系
@@ -22,6 +22,7 @@ import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import net.sf.json.JSONObject
import org.apache.commons.lang3.SerializationUtils
import org.apache.commons.lang3.StringUtils
import org.apache.tools.ant.util.DateUtils
import org.bson.types.ObjectId
@@ -92,7 +93,7 @@ class DataStandardBatch extends BaseBatch{
cityMap.each{city,alias->
println "역迦속潼±${city}×"
// if(city == "¸£ÖÝ"){
// if(city == "ÉϺ£"){
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
districtMap.putAll(detailMap)
// }
@@ -463,7 +464,7 @@ class DataStandardBatch extends BaseBatch{
// }catch (Exception e){
// addr = data.address;
// }
def copyObj = data
def copyObj = SerializationUtils.clone(data)
// copyObj.address = addr
district = getMatchAddressFromMap(copyObj)
if(district){
@@ -492,7 +493,7 @@ class DataStandardBatch extends BaseBatch{
}
if(!district){
def copyObj = data
def copyObj = SerializationUtils.clone(data)
copyObj.address = addr
district = getOpenApiMatch(copyObj)
if(!district && addr){
@@ -906,7 +907,13 @@ class DataStandardBatch extends BaseBatch{
def getOpenApiMatch(def data){
if(StringUtils.isNotEmpty(data.name)){
def jsonObject = OpenApiClient.addressMatch(data.city, data.name, "property", false, data.region, null, "fuzzy", false)
def jsonObject = null
if(DataUtil.isPreciseAddress(data.address)){
jsonObject = OpenApiClient.addressMatch(data.city, data.address, "property", false, data.region, data.name, "fuzzy", false)
}else{
jsonObject = OpenApiClient.addressMatch(data.city, data.name, "property", false, data.region, null, "fuzzy", false)
}
if(jsonObject && jsonObject.credit>0 ){
def dbo = [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address,estate_type:jsonObject.propertyType, credit:jsonObject.credit]
if(dbo){
@@ -923,7 +930,7 @@ class DataStandardBatch extends BaseBatch{
def findAllNoMatchData(){
DBObject qdata = new BasicDBObject();
qdata.put("s_date", new BasicDBObject('$gte', SHORT_DATE))
// qdata.put("city","¸£ÖÝ")
// qdata.put("city","ÉϺ£")
// qdata.put("source","갛앙와")
// qdata.put("name",Pattern.compile("묠소酪.*"))
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));
@@ -70,8 +70,16 @@ class SynchDistrictPrice extends BaseBatch{
data.put("estate_type",d.property_type)
data.put("s_date", SHORT_DATE)
data.put("url", d.url)
data.put("id",d.id)
data.put("ownership_type",d.ownership_type)
data.put("property_type",d.property_type)
data.put("build_type",d.build_type)
data.put("c_date", new Date())
data.put("m_date", new Date())
// if(d.location){
// data.put("lng", d.location.x)
// data.put("lat", d.location.y)
// }
rets << data
}
})
@@ -2,13 +2,15 @@ package com.fangjia.script.trade
import com.fangjia.core.util.DataUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
///导入价格指数
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
@@ -17,22 +19,37 @@ import org.apache.commons.lang.StringUtils
import org.apache.commons.lang.math.RandomUtils
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicLong
@BatchOwner(Watcher.GaoJuan)
class ImportIndexStats extends BaseBatch{
public static final def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
//public final static DBCollection under_line = new Mongo("192.168.0.136", 27017).getDB('fangjia').getCollection('stats_index');
public static final DBCollection on_line = MongoUtils.getCollectionByName("fangjia", "stats_index", "mongo.db.fangjia.com")
public static final DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName("fangjia", "region_stat", "mongo.db.fangjia.com")
//public static final DSCollection SEAWATER_COLL = new DataStore("192.168.0.65", 27777).getCollection("fangjia.seawater")
public static final DBCollection STATS_INDEX_COLL = MongoUtils.getCollectionByName("fangjia_stat", "stats_index", "mongo.db.fangjia.com")
public static final DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName2("fangjia", "region_stat", mongoDbUrl)
final DBCollection indices_price_house_sellColl = MongoUtils.getCollectionByName("crawl", "indices_price_house_sell", "192.168.0.206", 27020)
final DBCollection indices_price_house_sell_typeColl = MongoUtils.getCollectionByName("crawl", "indices_price_house_sell_type", "192.168.0.206", 27020)
final DBCollection indices_price_house_newColl = MongoUtils.getCollectionByName("crawl", "indices_price_house_new", "192.168.0.206", 27020)
final DBCollection indices_price_house_new_typeColl = MongoUtils.getCollectionByName("crawl", "indices_price_house_new_type", "192.168.0.206", 27020)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
//DSCollection seawater = new DataStore("192.168.0.65", 27777).getCollection("fangjia.seawater")
private static def cityList=["北京", "天津", "石家庄", "太原", "呼和浩特", "沈阳", "大连", "长春", "哈尔滨", "上海", "南京", "杭州", "宁波", "合肥", "福州", "厦门",
"南昌", "济南", "青岛", "郑州", "武汉", "长沙", "广州", "深圳", "南宁", "海口", "重庆", "成都", "贵阳", "昆明", "西安", "兰州", "西宁", "银川", "乌鲁木齐", "唐山",
"秦皇岛", "包头", "丹东", "锦州", "吉林", "牡丹江", "无锡", "扬州", "徐州", "温州", "金华", "蚌埠", "安庆", "泉州", "九江", "赣州", "烟台", "济宁", "洛阳", "平顶山",
"宜昌", "襄阳", "岳阳", "常德", "惠州", "湛江", "韶关", "桂林", "北海", "三亚", "泸州", "南充", "遵义", "大理"];
public static final def dingjiYear = 2016;
public static def dingjiYear = 2020;
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
private static final EsDataSource dataSource = new EsDataSource()
@@ -50,113 +67,283 @@ class ImportIndexStats extends BaseBatch{
@Override
public void process(Object args) {
// updateDataType()
// synch()
// synchTjjIndex()//批量同步统计局抓取过来的指数
Calendar calendar=Calendar.getInstance();
def month= calendar.get(Calendar.MONTH)+1;
def month= calendar.get(Calendar.MONTH);
def year= calendar.get(Calendar.YEAR);
def date = createDateString(year, month, 0, "");
calculatePriceIndex(year, month);
/*def s = "201601"
def day = calendar.get(Calendar.DATE)
if(day == 5){
calculatePriceIndex(year, month, date);
}
createTjjData(date)//同步单期统计局数据
// addLostIndex()
// addHistoryIndex()
}
def addLostIndex(){
def citys = CommonDatautils.findAllCity();
def index = 0
for(def city : citys){
index++
if(cityList.contains(city)){
println "[${index}] "+ city +" 数据来源于统计局"
continue;
}
def start = findStartOrEndIndex(city, 1)
def end = findStartOrEndIndex(city, -1)
if(start && end){
def startYearmonth = 201601
if(start.yearmonth>201601){
startYearmonth = start.yearmonth
}
def months = DateUtils.getBetweenMonth(startYearmonth.toString(), end.yearmonth.toString())
for(def i=0;i<months.size();i++){
if(i>0){
def dbo = findCityIndexByMonth(city, months[i].toInteger())
if(dbo && (!dbo.chain || !dbo.yeartoyear || !dbo.fixedbase)){
def lastMonthObj = findCityIndexByMonth(city, months[i-1])
def nextObj = findCityIndexByMonth(city, months[i+1])
def udata = new BasicDBObject()
if(!dbo.chain){
def indexs = []
if(lastMonthObj && lastMonthObj.chain){
indexs << lastMonthObj.chain
}
if(nextObj && nextObj.chain){
indexs << nextObj.chain
}
if(indexs){
udata.put("chain",DataHandleUtils.getDoubleRound(1, indexs.sum()/indexs.size()))
udata.put("remark","推导")
}
}
if(!dbo.yeartoyear){
def indexs = []
if(lastMonthObj && lastMonthObj.yeartoyear){
indexs << lastMonthObj.yeartoyear
}
if(nextObj && nextObj.yeartoyear){
indexs << nextObj.yeartoyear
}
if(indexs){
udata.put("yeartoyear",DataHandleUtils.getDoubleRound(1, indexs.sum()/indexs.size()))
udata.put("remark","推导")
}
}
if(!dbo.fixedbase){
def indexs = []
if(lastMonthObj && lastMonthObj.fixedbase){
indexs << lastMonthObj.fixedbase
}
if(nextObj && nextObj.fixedbase){
indexs << nextObj.fixedbase
}
if(indexs){
udata.put("fixedbase",DataHandleUtils.getDoubleRound(1, indexs.sum()/indexs.size()))
udata.put("remark","推导")
}
}
if(udata){
println city +" 补充"
STATS_INDEX_COLL.update(new BasicDBObject("_id", dbo._id), new BasicDBObject('$set', udata),false, true);
}
}
}
}
println "[${index}] "+ city +" 已处理完"
}
}
}
def findStartOrEndIndex(city , sort){
def q = new BasicDBObject("city", city)
def curosr = STATS_INDEX_COLL.find(q).sort(new BasicDBObject("yearmonth", sort)).limit(1)
def obj = null
for(def d : curosr){
obj = d
}
return obj;
}
def findCityIndexByMonth(city , month){
def q = new BasicDBObject("city", city).append("yearmonth", month)
return STATS_INDEX_COLL.findOne(q)
}
def addHistoryIndex(){
def s = "202001"
def startDate = null;
def stop = false;
def runDates = []
while(!stop){
startDate = DateUtils.parseFromTightYYYYMM(s);
def year = startDate.getYear()+1900;
def month = startDate.month+1;
def date = createDateString(year, month, 0, "");
def map = [:]
map.year = year
map.month = month
map.date = date
runDates << map
s = DateUtils.addMonth(date, 1, "yyyyMM")
if(year==2017 && month==7){
if(year==2024 && month==10){
stop = true;
}
calculatePriceIndex(year, month);
println "运行......"+date
}*/
}
//计算统计局没有发布的城市的价格指数
def calculatePriceIndex(def year, def month){
def citys =CommonDatautils.findAllCity();
def date = createDateString(year, month, 0, "");
def haveIndexCitys = findPriceIndexCitys(date);
def index = 0;
if(citys){
for(def city : citys){
if(cityList.contains(city)){
continue;
}
def id = findPriceIndex(city, date);
if(id){//已计算
continue;
}
def result = [:];
calculateAvgPriceIndex(city, year, month, result);
def curMonthPriceMap = findHouse2(city, createDateString(year, month, 1, "-"), createDateString(year, month + 1, 1, "-"));
def lastMonthPriceMap = findHouse2(city, createDateString(year, month-1, 1, "-"), createDateString(year, month , 1, "-"));
def lastYearPriceMap = findHouse2(city, createDateString(year-1, month, 1, "-"), createDateString(year-1, month + 1, 1, "-"));
def fixPriceMap = findHouse2(city, createDateString(dingjiYear, month, 1, "-"), createDateString(dingjiYear, month+1 , 1, "-"));
calculateAreaPriceInfex(result, 90, curMonthPriceMap.get("0_90"),lastMonthPriceMap.get("0_90"),lastYearPriceMap.get("0_90"), fixPriceMap.get("0_90"));//0-90
calculateAreaPriceInfex(result, 144, curMonthPriceMap.get("90-144"),lastMonthPriceMap.get("90-144"),lastYearPriceMap.get("90-144"), fixPriceMap.get("90-144"));//90-144
calculateAreaPriceInfex(result, 1000, curMonthPriceMap.get("144_500"),lastMonthPriceMap.get("144_500"),lastYearPriceMap.get("144_500"), fixPriceMap.get("144_500"));//144-1000
// calculateAreaPriceInfex(city, year, month, result, 0, 90);//0-90
// calculateAreaPriceInfex(city, year, month, result, 90, 144);//90-144
// calculateAreaPriceInfex(city, year, month, result, 144, 1000);//144-1000
checkPriceIndex(result);
index++;
if(result){
result.city = city;
result.yearmonth = date;
result.source = 2;
result.remark = "均价计算";
result.cat = "district";
savePriceIndex(result);
}
println "${index}"+ result;
}
}
for(def map : runDates){
calculatePriceIndex(map.year, map.month, map.date);
}
}
//计算统计局没有发布的城市的价格指数
def calculatePriceIndex(def year, def month, def date){
// def citys = ["定安","五指山","琼海","儋州","文昌","万宁","东方","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]//
def citys = CommonDatautils.findAllCity();
// def date = createDateString(year, month, 0, "");
def haveIndexCitys = findPriceIndexCitys(date.toInteger());
def totalCount = new AtomicLong()
def addCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(citys.size())
def newCitys = []
for(def i=0;i<citys.size();i++) {
def city = citys[i]
if (!haveIndexCitys.contains(city) && !cityList.contains(city)) {
newCitys << city
}
}
def total = newCitys.size()
for(def i=0;i<newCitys.size();i++){
final def city = newCitys[i]
def id = findPriceIndex(city, date.toInteger());
if(id){//已计算
continue;
}
if(date.toInteger()>=203101){
dingjiYear = 2030
}else if(date.toInteger()>=202601){
dingjiYear = 2025
}else if(date.toInteger()>=202101){
dingjiYear = 2020
}else if(date.toInteger()>=201601){
dingjiYear = 2015
}
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def result = calculating(city, year, month, date);
println result
println "${city}[${totalCount.get()}/${total}]运行......"+year+"-"+month+" 完成"
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await(1, TimeUnit.MINUTES);
// latch.await(1, TimeUnit.MINUTES);
}
def calculating(city, year, month, date){
def result = [:];
calculateAvgPriceIndex(city, year, month, result);
if(result){
calculateAreaPriceInfex(result, city, year, month, 90);//0-90
calculateAreaPriceInfex(result, city, year, month, 144);//90-144
calculateAreaPriceInfex(result, city, year, month, 1000);//144-1000
checkPriceIndex(result);//涨跌幅控制
if(result){
result.city = city;
result.yearmonth = date.toInteger();
result.source = 2;
result.remark = "均价计算";
result.cat = "district";
result.type = "城市"
savePriceIndex(result);
}
}
return result
}
//计算新建住房的价格指数
//计算住房的价格指数
def calculateAvgPriceIndex(def city, def year, def month, def result){
def curMonthPrice = findCityByDates(city, createDateString(year, month, 1, "-"), createDateString(year, month + 1, 1, "-"));
def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMM(year.toString()+month.toString())),"yyyy-MM-dd")
def endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMM(year.toString()+(month.toString()))),"yyyy-MM-dd")
def curMonthPrice = findCityByDates(city, startDate, endDate);
// def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMM(year.toString()+month.toString())),"yyyyMMdd")
// def endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMM(year.toString()+(month.toString()))),"yyyyMMdd")
// def curMonthPrice = findHouse3(city, startDate.toInteger(), endDate.toInteger(), null)
float markup = 0f;
float yearMarkup = 0f;
float dingJiMarkup = 0f;
//环比涨跌幅
def lastMonthPrice = findCityByDates(city, createDateString(year, month - 1, 1, "-"), createDateString(year, month, 1, "-"));
if(lastMonthPrice){
startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMM(year.toString()+month.toString()),-1),"yyyy-MM-dd")
endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMM(year.toString()+month.toString()),-1),"yyyy-MM-dd")
def lastMonthPrice = findCityByDates(city, startDate, endDate);
// startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMM(year.toString()+month.toString()),-1),"yyyyMMdd")
// endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMM(year.toString()+month.toString()),-1),"yyyyMMdd")
// def lastMonthPrice = findHouse3(city, startDate.toInteger(), endDate.toInteger(), null)
if(curMonthPrice && lastMonthPrice){
markup = curMonthPrice *100/lastMonthPrice
markup = DataUtil.getDoubleRound(1,markup);
}else{
def v = random(-3, 3) + 100;
markup = v;
}
// else{
// def v = random(-3, 3) + 100;
// markup = v;
// }
//同比涨跌幅
def lastYearPrice = findCityByDates(city, createDateString(year-1, month, 1, "-"), createDateString(year-1, month+1 , 1, "-"));
if(lastYearPrice){
startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMM((year-1).toString()+month.toString())),"yyyy-MM-dd")
endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMM((year-1).toString()+(month.toString()))),"yyyy-MM-dd")
def lastYearPrice = findCityByDates(city, startDate, endDate);
// startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMM((year-1).toString()+month.toString())),"yyyyMMdd")
// endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMM((year-1).toString()+(month.toString()))),"yyyyMMdd")
// def lastYearPrice = findHouse3(city, startDate.toInteger(), endDate.toInteger(), null)
if(curMonthPrice && lastYearPrice){
yearMarkup = curMonthPrice * 100/lastYearPrice;
yearMarkup = DataUtil.getRound(1,yearMarkup);
}else{
def v = random(-5, 5) + 100;
yearMarkup = v;
}
// else{
// def v = random(-5, 5) + 100;
// yearMarkup = v;
// }
//定基涨跌幅
def dingJiPrice = findCityByDates(city, createDateString(dingjiYear, month, 1, "-"), createDateString(dingjiYear, month+1 , 1, "-"));
if(dingJiPrice){
startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD2(dingjiYear.toString()+"0101")),"yyyy-MM-dd")
endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD2(dingjiYear.toInteger()+"1231")),"yyyy-MM-dd")
def dingJiPrice = findCityByDates(city, startDate, endDate);
// startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD2(dingjiYear.toString()+"0101")),"yyyyMMdd")
// endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD2(dingjiYear.toInteger()+"1231")),"yyyyMMdd")
// def dingJiPrice = findHouse3(city, startDate, endDate, null)
if(curMonthPrice && dingJiPrice){
dingJiMarkup = curMonthPrice*100/dingJiPrice;
dingJiMarkup = DataUtil.getRound(1,dingJiMarkup);
}else{
def v = random(-8, 8) + 100;
dingJiMarkup = v;
}
// else{
// def v = random(-8, 8) + 100;
// dingJiMarkup = v;
// }
if(markup){
result.markup = markup;
@@ -173,45 +360,53 @@ class ImportIndexStats extends BaseBatch{
return result;
}
//计算住宅按平米分类的价格指数
def calculateAreaPriceInfex(def result, def endArea, def curHousePrices1, def lastMonthHousePrices1, def lastYearHousePrice2, def dingjiHousePrices3){
//def curHousePrices1 = findHouse(city, startArea, endArea, createDateString(year, month, 1, "-"), createDateString(year, month + 1, 1, "-"));
def cur_price = getAvgPrice(curHousePrices1);
def calculateAreaPriceInfex(def result, def city, def year, def month, def endArea){
def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMM(year.toString()+month.toString())),"yyyyMMdd")
def endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMM(year.toString()+(month.toString()))),"yyyyMMdd")
def cur_price = findHouse3(city, startDate, endDate, endArea);
def areaMarkup = 0;
def areaYearMarkup = 0;
def areaDingJiMarkup = 0;
//环比涨跌幅
//def lastMonthHousePrices1 = findHouse(city, startArea, endArea, createDateString(year, month-1, 1, "-"), createDateString(year, month , 1, "-"));
def lastMonthlt90_price = getAvgPrice(lastMonthHousePrices1);
startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMM(year.toString()+month.toString()),-1),"yyyyMMdd")
endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMM(year.toString()+month.toString()),-1),"yyyyMMdd")
def lastMonthlt90_price = findHouse3(city, startDate, endDate, endArea);
if(cur_price && lastMonthlt90_price){
areaMarkup = cur_price*100/lastMonthlt90_price
areaMarkup = DataUtil.getRound(1,areaMarkup);
}else{
def v = random(-3, 3) + 100;
areaMarkup = v;
}
// else{
// def v = random(-3, 3) + 100;
// areaMarkup = v;
// }
//def lastYearHousePrice2 = findHouse(city, startArea, endArea, createDateString(year-1, month, 1, "-"), createDateString(year-1, month + 1, 1, "-"));
def lastYearprice = getAvgPrice(lastYearHousePrice2);
startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMM((year-1).toString()+month.toString())),"yyyyMMdd")
endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMM((year-1).toString()+(month.toString()))),"yyyyMMdd")
def lastYearprice = findHouse3(city, startDate, endDate, endArea);
//同比涨跌幅
if(cur_price && lastYearprice){
areaYearMarkup = cur_price *100/lastYearprice
areaYearMarkup = DataUtil.getRound(1,areaYearMarkup);
}else{
def v = random(-5, 5) + 100;
areaYearMarkup = v;
}
// else{
// def v = random(-5, 5) + 100;
// areaYearMarkup = v;
// }
//定基涨跌幅
//def dingjiHousePrices3 = findHouse(city, startArea, endArea, createDateString(2010, month, 1, "-"), createDateString(2010, month+1 , 1, "-"));
def dingJiPrice = getAvgPrice(dingjiHousePrices3);
startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD2(dingjiYear.toString()+"0101")),"yyyyMMdd")
endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD2(dingjiYear.toInteger()+"1231")),"yyyyMMdd")
def dingJiPrice = findHouse3(city, startDate, endDate, endArea);
if(cur_price && dingJiPrice){
areaDingJiMarkup = cur_price *100/dingJiPrice
areaDingJiMarkup = DataUtil.getRound(1,areaDingJiMarkup);
}else{
def v = random(-8, 8) + 100;
areaDingJiMarkup = v;
}
// else{
// def v = random(-8, 8) + 100;
// areaDingJiMarkup = v;
// }
if(areaMarkup){
@@ -236,18 +431,18 @@ class ImportIndexStats extends BaseBatch{
result.each {k,v->
def temp = v - 100;
if(k == "markup" || k == "areaMarkup90" || k == "areaMarkup144" || k == "areaMarkup1000"){
if(temp < -2 || temp > 1){
v = getNewMarkup(temp, -2, 1) + 100;
if(temp < -3 || temp > 3){
v = getNewMarkup(temp, -3, 3) + 100;
detailFlag = 2;
}
}else if(k == "yearMarkup" || k == "areaYearMarkup90" || k == "areaYearMarkup144" || k == "areaYearMarkup1000"){
if(temp < -5 || temp > 7){
v = getNewMarkup(temp, -5, 7) + 100;
if(temp < -20 || temp > 30){
v = getNewMarkup(temp, -20, 30) + 100;
detailFlag = 2;
}
}else{
if(temp < -6 || temp > 29){
v = getNewMarkup(temp, -6, 29) + 100;
if(temp < -50 || temp > 100){
v = getNewMarkup(temp, -50, 100) + 100;
detailFlag = 2;
}
}
@@ -320,27 +515,56 @@ class ImportIndexStats extends BaseBatch{
// def dbCursor = seawater.find(q, keys).batchSize(5000);
// Iterator<BasicDBObject> it = dbCursor.iterator()
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
println city +" : "+startDate+" "+endDate +" "+startArea+"-"+endArea
def prices = [];
while (rs.next()) {
ElasticUtils.executeSqlQuery(sqlBuffer.toString(),{ rs->
def area = (Double) rs.getObject("area")
def totalPrice = (Integer) rs.getObject("total_price")
if(!totalPrice){
continue;
}
if(area <= 25 && area>=500){
continue;
}
def price = totalPrice.toInteger()/area.toDouble();
if(price > 1000 && price < 200000){
prices << price.toInteger()
}
}
})
return prices;
}
def findHouse3(def city,def startDate, def endDate, def endArea){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT avg(avg_price) as avg_price FROM \"seawater-20*\" WHERE 1=1")
sqlBuffer.append(" and city = '").append(city).append("'")
sqlBuffer.append(" and cat = 'sell'")
sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <="+endDate);
sqlBuffer.append(" and avg_price < 300000")
if(endArea){
if(endArea ==90){
sqlBuffer.append(" and area>=25 and area<90 and total_price>0")
}else if (endArea == 144){
sqlBuffer.append(" and area>=90 and area<144 and total_price>0 ")
}else if (endArea > 144){
sqlBuffer.append(" and area>=144 and total_price>0 ")
}
}
def avg_price = null;
try{
ElasticUtils.executeSqlQuery(sqlBuffer.toString(),{ rs->
avg_price = rs.avg_price
})
}catch(Exception e){
avg_price = findHouse3(city,startDate, endDate, endArea);
println e.getMessage()
}
if(avg_price){
return (int)avg_price;
}
return null
}
@@ -367,21 +591,14 @@ class ImportIndexStats extends BaseBatch{
}
sqlBuffer.append(" and cat = 'sell'")
sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <="+endDate);
sqlBuffer.append(" ORDER BY p_date desc limit 5000")
sqlBuffer.append(" and area>=25 and area<=500 and total_price>0 ORDER BY p_date desc limit 5000")
def priceMap = [:];
try{
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
while (rs.next()) {
def area = (Double) rs.getObject("area")
def totalPrice = (Integer) rs.getObject("total_price")
if(!totalPrice){
continue;
}
if(area <= 25 && area>=500){
continue;
}
ElasticUtils.executeSqlQuery(sqlBuffer.toString(),{ rs->
def area = rs.area
def totalPrice = rs.total_price
def areaKey = "";
if(area>0 && area<90){
areaKey = "0_90";
@@ -403,7 +620,7 @@ class ImportIndexStats extends BaseBatch{
prices << price.toInteger()
}
}
})
}catch(Exception e){
findHouse2(city,startDate, endDate);
}
@@ -428,8 +645,7 @@ class ImportIndexStats extends BaseBatch{
def findCityByDates(def city, def startDate, def endDate){
DBObject q = new BasicDBObject()
q.put("city", city);
q.put("region", null);
q.put("block", null);
q.put("cat", "城市")
if(startDate && endDate){
q.put('weekend', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lte', DateUtils.parseFromTightYYYYMMDD(endDate)));
}
@@ -471,10 +687,12 @@ class ImportIndexStats extends BaseBatch{
def findPriceIndexCitys(def date){
DBObject q = new BasicDBObject();
q.put("yearmonth", date);
q.put("cat", "district")
q.put("type", "城市")
DBObject keys = new BasicDBObject();
keys.put("city", 1)
def cursor = on_line.find(q, keys);
def cursor = STATS_INDEX_COLL.find(q, keys);
def citys = [];
for(def dbo : cursor){
if(!citys.contains(dbo.city)){
@@ -489,11 +707,12 @@ class ImportIndexStats extends BaseBatch{
DBObject q = new BasicDBObject();
q.put("yearmonth", date);
q.put("city", city);
q.put("cat", "district");
q.put("type", "城市")
DBObject keys = new BasicDBObject();
keys.put("city", 1)
def cursor = on_line.find(q, keys);
def citys = [];
def cursor = STATS_INDEX_COLL.find(q, keys);
for(def dbo : cursor){
def id = dbo.get("_id");
return id;
@@ -508,15 +727,16 @@ class ImportIndexStats extends BaseBatch{
dbo.put("source",result.source);
dbo.put("remark",result.remark);
dbo.put("cat",result.cat);
dbo.put("type",result.type);
dbo.put("c_date",new Date());
if(result.markup){
dbo.put("new_chain", result.markup.toDouble());
dbo.put("chain", result.markup.toDouble());
}
if(result.yearMarkup){
dbo.put("new_yeartoyear", result.yearMarkup.toDouble());
dbo.put("yeartoyear", result.yearMarkup.toDouble());
}
if(result.dingJiMarkup){
dbo.put("new_fixedbase", result.dingJiMarkup.toDouble());
dbo.put("fixedbase", result.dingJiMarkup.toDouble());
}
if(result.areaMarkup90){
@@ -560,12 +780,158 @@ class ImportIndexStats extends BaseBatch{
if(id){
DBObject q = new BasicDBObject()
q.put("_id", id)
on_line.update(q, new BasicDBObject('$set', dbo));
STATS_INDEX_COLL.update(q, new BasicDBObject('$set', dbo));
}else{
on_line.save(dbo);
STATS_INDEX_COLL.save(dbo);
}
}
def updateDataType(){
def cursor = on_line.find(new BasicDBObject()).batchSize(5000)
def index = 0
for(def d : cursor){
index++
def udata = new BasicDBObject()
udata.put("yearmonth", d.yearmonth.toInteger())
on_line.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
println index
}
}
def synch(){
def cursor = on_line.find(new BasicDBObject()).batchSize(5000)
for(def d : cursor){
STATS_INDEX_COLL.save(d)
}
}
def synchTjjIndex(){
def s = "202401"
def startDate = null;
def stop = false;
while(!stop){
startDate = DateUtils.parseFromTightYYYYMM(s);
def year = startDate.getYear()+1900;
def month = startDate.month+1;
def date = createDateString(year, month, 0, "");
createTjjData(date)
println "运行......"+date+".......保存完成"
s = DateUtils.addMonth(date, 1, "yyyyMM")
if(year==2024 && month==8){
stop = true;
}
}
}
def createTjjData(date){
def count = STATS_INDEX_COLL.count(new BasicDBObject("type","城市").append("source", 1).append("yearmonth", date.toInteger()))
if(count == 0){
def queryStartDate = DateUtils.parseFromTightByPattern(date+"01", "yyyyMMdd")
def queryEndDate = DateUtils.parseFromTightByPattern(date+"05", "yyyyMMdd")
def sellIndexMap = findTjjIndex(indices_price_house_sellColl, queryStartDate, queryEndDate)
def sellIndexAreaMap = findTjjAreaIndex(indices_price_house_sell_typeColl, queryStartDate, queryEndDate)
def newIndexMap = findTjjIndex(indices_price_house_newColl, queryStartDate, queryEndDate)
def newAreaIndexAreaMap = findTjjAreaIndex(indices_price_house_new_typeColl, queryStartDate, queryEndDate)
def rets = []
sellIndexMap.each{city, d->
def dbo = new BasicDBObject()
dbo.put("city",d.city)
dbo.put("yearmonth",date.toInteger())
dbo.put("source" ,1)
dbo.put("remark" , "统计局数据导入")
dbo.put("cat" , "district")
dbo.put("c_date" , new Date())
dbo.put("m_date" , new Date())
dbo.put("type" , "城市")
dbo.put("chain" ,d.mom)
dbo.put("yeartoyear" , d.yoy)
dbo.put("fixedbase" , d.fixedbase)
def areaMap = sellIndexAreaMap.get(d.city)
def areaObj1 = areaMap.get("90")
dbo.put("below_chain" , areaObj1.mom)
dbo.put("below_yeartoyear" , areaObj1.yoy)
dbo.put("below_fixedbase" , areaObj1.fixedbase)
def areaObj2 = areaMap.get("90-144")
dbo.put("among_chain" , areaObj2.mom)
dbo.put("among_yeartoyear" , areaObj2.yoy)
dbo.put("among_fixedbase" , areaObj2.fixedbase)
def areaObj3 = areaMap.get("144")
dbo.put("above_chain" , areaObj3.mom)
dbo.put("above_yeartoyear" , areaObj3.yoy)
dbo.put("above_fixedbase" , areaObj3.fixedbase)
def newObj = newIndexMap.get(d.city)
dbo.put("new_chain" , newObj.mom)
dbo.put("new_yeartoyear" , newObj.yoy)
dbo.put("new_fixedbase" , newObj.fixedbase)
def newAreaMap = newAreaIndexAreaMap.get(d.city)
def areaNewObj1 = newAreaMap.get("90")
dbo.put("new_below_chain" , areaNewObj1.mom)
dbo.put("new_below_yeartoyear" , areaNewObj1.yoy)
dbo.put("new_below_fixedbase" , areaNewObj1.fixedbase)
def areaNewObj2 = newAreaMap.get("90-144")
dbo.put("new_among_chain" , areaNewObj2.mom)
dbo.put("new_among_yeartoyear" , areaNewObj2.yoy)
dbo.put("new_among_fixedbase" , areaNewObj2.fixedbase)
def areaNewObj3 = newAreaMap.get("144")
dbo.put("new_above_chain" , areaNewObj3.mom)
dbo.put("new_above_yeartoyear" , areaNewObj3.yoy)
dbo.put("new_above_fixedbase" , areaNewObj3.fixedbase)
rets << dbo
}
if(rets){
STATS_INDEX_COLL.insert(rets)
}
}
}
def findTjjIndex(coll, queryStartDate, queryEndDate){
def q = new BasicDBObject()
q.put("date", new BasicDBObject('$gte',queryStartDate).append('$lte', queryEndDate))
def dbcursor = coll.find(q)
def map = [:]
for(def d : dbcursor){
def o = map.get(d.city)
if(!o){
map.put(d.city,d)
}
}
return map;
}
def findTjjAreaIndex(coll,queryStartDate, queryEndDate){
def q = new BasicDBObject()
q.put("date", new BasicDBObject('$gte',queryStartDate).append('$lte', queryEndDate))
def dbcursor = coll.find(q)
def map = [:]
for(def d : dbcursor){
def areaMap = map.get(d.city)
if(!areaMap){
areaMap = [:]
map.put(d.city, areaMap)
}
def o = areaMap.get(d.area)
if(!o){
areaMap.put(d.area, d)
}
}
return map;
}
}
@@ -99,7 +99,7 @@ class BaiduMapUtils {
public static JSONObject searchByAddress(String address) {
String add = URLEncoder.encode(address, "utf-8");
String url = "http://api.map.baidu.com/geocoding/v3/?address="+ add +"&output=json&ak=9E99D3540e4dafb7894f7b9fe27fa6b5"
String url = "http://api.map.baidu.com/geocoding/v3/?address="+ add +"&output=json&ak=MZpy11ZrE8CbG4M5OznVAxIcRaBZ7aTR"
boolean stop = false;
int recount = 0;
while(!stop && recount<3) {
@@ -108,6 +108,7 @@ class BaiduMapUtils {
try{
String result = request.get(url,null,"UTF-8");
JSONObject json = JSONObject.fromObject(result);
println address +" "+ json
if(json.getInt("status") >= 300){
changeAK();
json = searchByAddress(address);
@@ -988,6 +988,28 @@ class DateUtils {
return null;
}
public static List<String> getBetweenMonth(start, end){
def stop = false;
def months = []
while(!stop){
start = com.fangjia.script.util.DateUtils.parseFromTightYYYYMM(start);
def year = start.getYear()+1900;
def month = start.month+1;
if(month <10){
start = year.toString()+"0"+month.toString()
}else{
start = year.toString()+month.toString()
}
months << start
start = DateUtils.addMonth(start, 1, "yyyyMM")
if(start > end){
stop = true;
}
}
return months;
}
public static long random(long begin, long end) {
long rtn = begin + (long) (Math.random() * (end - begin));
// 如果返回的是开始时间和结束时间,则递归调用本函数查找随机值
@@ -414,7 +414,7 @@ public class ExcelUtils{
def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
def body = data.subList(1, data.size())
def nThread = Runtime.getRuntime().availableProcessors() * 4
def nThread = Runtime.getRuntime().availableProcessors() * 6
println nThread
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
final AtomicInteger counter = new AtomicInteger(0)
@@ -40,7 +40,7 @@ class GaoDeApiUtils {
def static geo(city, address) {
Map<String, String> params = new HashMap<String, String>()
params.put("key", GAO_DE_AK)
params.put("key", "a34894d151fffff94f9a33bae3d5565c")//ÐíéF½Ü
params.put("city", city)
params.put("address", address)
String content