update
This commit is contained in:
+105
-17
@@ -1,11 +1,18 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 将国信达及成交对标数据加入到第三方均价表,做为均价参考
|
||||
*/
|
||||
@@ -17,18 +24,24 @@ class AddThirdPriceSample {
|
||||
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
static main(args) {
|
||||
def task = new AddThirdPriceSample()
|
||||
task.run() //添加第三方 fccs->保存的是国信达的参考价格 csjj->保存的是国信达的成交数据
|
||||
// task.addTrade() //添加成交
|
||||
// task.run() //添加第三方 fccs->保存的是国信达的参考价格 csjj->保存的是国信达的成交数据
|
||||
task.addTrade() //添加成交
|
||||
|
||||
// task.matchOldData()
|
||||
// task.noInCcbCity()
|
||||
|
||||
}
|
||||
def run(){
|
||||
for(def i=0;i<19;i++){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\国信达参考价作为第三方入库.xlsx",{d->
|
||||
for(def i=0;i<1;i++){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\国信达指导价录入-1024.xlsx",{d->
|
||||
if(d['A'] && d['B']){
|
||||
def dist = CommonDatautils.findDistrictById(d['A'].toString().trim())
|
||||
if(dist){
|
||||
def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202408)
|
||||
def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202409)
|
||||
// def udata = new BasicDBObject("csjj_esf_price", d['C'].toString().toDouble().toInteger())
|
||||
def udata = new BasicDBObject("fccs_esf_price", d['B'].toString().toDouble().toInteger())
|
||||
AVERAGE_STATISTICS.update(q,new BasicDBObject('$set', udata),false, false)
|
||||
@@ -44,15 +57,16 @@ class AddThirdPriceSample {
|
||||
}
|
||||
|
||||
def addTrade(){
|
||||
def citys = ["武汉","南京","杭州","长沙","郑州","合肥","宁波","南通","嘉兴","珠海","泉州","绍兴","湖州","镇江"]
|
||||
def rets = findTrades(citys)
|
||||
// DEAL_PRICE_COLL.insert(rets)
|
||||
// def citys = ["武汉","南京","杭州","长沙","郑州","合肥","宁波","南通","嘉兴","珠海","泉州","绍兴","湖州","镇江"]
|
||||
def rets = findTrades()
|
||||
DEAL_PRICE_COLL.insert(rets)
|
||||
}
|
||||
|
||||
def findTrades(citys){
|
||||
def findTrades(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", new BasicDBObject('$in', citys))
|
||||
q.put("flag", 1)
|
||||
q.put("ccb_price_remark","建行无此城市成交")
|
||||
// q.put("city", new BasicDBObject('$in', citys))
|
||||
// q.put("flag", 1)
|
||||
def dbcursor = NO_PRICE_DEAL_PRICE_COLL.find(q)
|
||||
def rets = []
|
||||
def index = 0
|
||||
@@ -62,7 +76,7 @@ class AddThirdPriceSample {
|
||||
|
||||
udata.put("_id",db._id)
|
||||
udata.put("url",db.url)
|
||||
udata.put("area",db.area)
|
||||
udata.put("area",db.area.toString().toDouble())
|
||||
udata.put("avg_price",db.avg_price.toString().toDouble().toInteger())
|
||||
udata.put("build_structure",db.build_structure)
|
||||
udata.put("build_type",db.build_type)
|
||||
@@ -73,8 +87,14 @@ class AddThirdPriceSample {
|
||||
udata.put("elevator",db.elevator)
|
||||
udata.put("fitment",db.fitment)
|
||||
udata.put("floor_info",db.floor_info)
|
||||
udata.put("hall",db.hall.toString().toInteger())
|
||||
udata.put("height",db.height.toString().toInteger())
|
||||
if(db.hall){
|
||||
udata.put("hall",db.hall.toString().toInteger())
|
||||
}
|
||||
|
||||
if(db.height){
|
||||
udata.put("height",db.height.toString().toInteger())
|
||||
}
|
||||
|
||||
udata.put("housing_age",db.housing_age)
|
||||
udata.put("lift",db.lift.toString().toInteger())
|
||||
udata.put("lift_household",db.lift_household.toString().toInteger())
|
||||
@@ -83,7 +103,7 @@ class AddThirdPriceSample {
|
||||
udata.put("room", db.room.toString().toInteger())
|
||||
udata.put("room_structure",db.room_structure)
|
||||
udata.put("source",db.source)
|
||||
udata.put("total_price",db.total_price.toString().toInteger())
|
||||
udata.put("total_price",db.total_price.toString().toDouble().toInteger())
|
||||
udata.put("trade_date",db.trade_date)
|
||||
udata.put("trade_property",db.trade_property)
|
||||
udata.put("house_using",db.house_using)
|
||||
@@ -98,11 +118,79 @@ class AddThirdPriceSample {
|
||||
udata.put("match_id",db.match_id)
|
||||
udata.put("price_flag", "国信达成交-推导")
|
||||
|
||||
DEAL_PRICE_COLL.save(udata)
|
||||
// DEAL_PRICE_COLL.save(udata)
|
||||
println index
|
||||
// rets << udata
|
||||
rets << udata
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
def matchOldData(){
|
||||
def q = new BasicDBObject("source","贝壳找房").append("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2024-09-01","yyyy-MM-dd")))
|
||||
def list = DEAL_PRICE_COLL.find(q).toList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(list.size())
|
||||
for(def i=0;i<list.size();i++){
|
||||
final def d = list[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def query = new BasicDBObject("url", d.url)
|
||||
def obj = NO_PRICE_DEAL_PRICE_COLL.findOne(query)
|
||||
if(obj){
|
||||
def update = new BasicDBObject()
|
||||
update.put("price_remark", "有价格")
|
||||
NO_PRICE_DEAL_PRICE_COLL.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set', update), false,false)
|
||||
}
|
||||
println totalCount.get()
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
def noInCcbCity(){
|
||||
def citys = ["大理","苏州","无锡","通辽","绵阳","吉安","潍坊","襄阳","济宁","张家口","泸州","乐山","赣州","九江","眉山","桂林","雅安","广州","景德镇","上海","运城","内江","金华","濮阳","丹东","淄博","新乡","廊坊","烟台","邯郸","萍乡","宜宾","咸阳","许昌","德阳","银川","洛阳","天津","湛江","长沙","深圳","湘西","信阳","成都","威海","福州","上饶","常州","宜春","天水","平顶山","常德","吉林","盐城","杭州","赤峰","南京","周口","济南","株洲","大连","徐州","泰安","太原","凉山","兰州","承德","广元","台州","南昌","乌鲁木齐","海口","北京","淮安","清远","临沂","柳州","衢州","攀枝花","呼和浩特","昆明","岳阳","衡阳","温州","石家庄","连云港","保定","阜阳","遂宁","菏泽","青岛","唐山","开封","漳州","贵阳","驻马店","宝鸡","达州","汉中","南充","遵义","包头","南宁","沈阳","重庆","芜湖","厦门","大同"]
|
||||
def q = new BasicDBObject()
|
||||
q.put("price_remark",new BasicDBObject('$ne', "有价格"))
|
||||
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2024-09-01","yyyy-MM-dd")))
|
||||
q.put("avg_price",new BasicDBObject('$gt', 0))
|
||||
q.put("match_id",new BasicDBObject('$exists', true))
|
||||
def list = NO_PRICE_DEAL_PRICE_COLL.find(q).toList()
|
||||
println "共:"+list.size()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(list.size())
|
||||
for(def i=0;i<list.size();i++){
|
||||
final def d = list[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
|
||||
if(!citys.contains(d.city)){
|
||||
def update = new BasicDBObject()
|
||||
update.put("ccb_price_remark", "建行无此城市成交")
|
||||
NO_PRICE_DEAL_PRICE_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set', update), false,false)
|
||||
}
|
||||
println totalCount.get()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+35
-32
@@ -1234,15 +1234,7 @@ def findThirdCommCount(cityid, sqlInstance){
|
||||
//}
|
||||
|
||||
//
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\无价格小区(3)2.23.xlsx"
|
||||
//ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
// if(dist){
|
||||
// d['G'] = dist.city
|
||||
// d['H'] = dist.region
|
||||
// d['I'] = dist.name
|
||||
// }
|
||||
//})
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -1331,34 +1323,45 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
|
||||
|
||||
|
||||
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\6个城市待优化_9月xlsx.xlsx"
|
||||
//
|
||||
ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
if(d['G']){
|
||||
// def region = RegionData.formatRegion(d['A'], d['J'])
|
||||
// if(region){
|
||||
// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district")
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\6个城市待优化_9月xlsx.xlsx"
|
||||
////
|
||||
//ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
// if(d['G']){
|
||||
//// def region = RegionData.formatRegion(d['A'], d['J'])
|
||||
//// if(region){
|
||||
//// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district")
|
||||
//// if(dist){
|
||||
//// d['B'] = dist.region
|
||||
//// d['C'] = dist.name
|
||||
//// d['L'] = dist.status-dist.name_cat
|
||||
//// }
|
||||
//// }
|
||||
// if(d['G'] instanceof String){
|
||||
// def dist = CommonDatautils.findDistrictById(d['G'])
|
||||
// if(dist){
|
||||
// d['B'] = dist.region
|
||||
// d['C'] = dist.name
|
||||
// d['L'] = dist.status-dist.name_cat
|
||||
//// d['A'] = dist.city
|
||||
//// d['B'] = dist.region
|
||||
//// d['C'] = dist.name
|
||||
//// d['K'] = dist.estate_type2
|
||||
//// d['L'] = dist.status
|
||||
//// d['M'] = dist.name_cat
|
||||
//
|
||||
// d['H'] = dist.name
|
||||
// }
|
||||
// }
|
||||
if(d['G'] instanceof String){
|
||||
def dist = CommonDatautils.findDistrictById(d['G'])
|
||||
if(dist){
|
||||
// d['A'] = dist.city
|
||||
// d['B'] = dist.region
|
||||
// d['C'] = dist.name
|
||||
// d['K'] = dist.estate_type2
|
||||
// d['L'] = dist.status
|
||||
// d['M'] = dist.name_cat
|
||||
// }
|
||||
//},0)
|
||||
|
||||
d['H'] = dist.name
|
||||
}
|
||||
}
|
||||
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20241025人工指导价.xlsx"
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
def dist = CommonDatautils.findDistrictById(d['J'])
|
||||
if(dist){
|
||||
d['A'] = dist.city
|
||||
d['B'] = dist.region
|
||||
d['C'] = dist.name
|
||||
}
|
||||
},0)
|
||||
})
|
||||
|
||||
|
||||
//def file = "D:\\数据\\建行\\房产估值\\指导价\\5城价格翻新6-21.xlsx"
|
||||
|
||||
+66
-1
@@ -4785,7 +4785,6 @@ class CommonDatautils {
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","区域")
|
||||
q.put("district_diff",1)
|
||||
q.put("city",new BasicDBObject('$in', ["深圳市"]))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("province",1)
|
||||
@@ -4814,6 +4813,72 @@ class CommonDatautils {
|
||||
return regionMap
|
||||
}
|
||||
|
||||
|
||||
static def queryAllRegionNameByCity(){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","区域")
|
||||
q.put("district_diff",1)
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city_short",1)
|
||||
keys.put("district_short",1)
|
||||
keys.put("alias",1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def retMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def regionMap = retMap.get(d.city_short)
|
||||
if(!regionMap){
|
||||
regionMap = [:]
|
||||
retMap.put(d.city_short, regionMap)
|
||||
}
|
||||
regionMap.put(d.district_short, d.alias)
|
||||
}
|
||||
|
||||
return retMap
|
||||
}
|
||||
|
||||
static def queryAllRegionFullNameByCity(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","区域")
|
||||
q.put("district_diff",1)
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city",1)
|
||||
keys.put("district",1)
|
||||
keys.put("alias",1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def retMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def regionMap = retMap.get(d.city)
|
||||
if(!regionMap){
|
||||
regionMap = [:]
|
||||
retMap.put(d.city, regionMap)
|
||||
}
|
||||
regionMap.put(d.district, d.alias)
|
||||
}
|
||||
|
||||
return retMap
|
||||
}
|
||||
|
||||
static def queryAllStreetNameByCity(city){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","乡镇街道")
|
||||
q.put("city", city)
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("street",1)
|
||||
keys.put("alias",1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def retMap = [:]
|
||||
for(def d : dbcursor){
|
||||
retMap.put(d.street, d.alias)
|
||||
}
|
||||
|
||||
return retMap
|
||||
}
|
||||
|
||||
static def queryCommunityByCity(){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
|
||||
+1
-1
@@ -49,7 +49,7 @@ class AutoAddDistrictBatch extends BaseBatch{
|
||||
|
||||
def fileName = "/home/goojia/gj/add_dist_info.xlsx"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\数据建设\\名录集\\第三方待入库数据.xlsx"
|
||||
fileName = "D:\\数据\\数据建设\\小区\\第三方待入库数据.xlsx"
|
||||
}
|
||||
ExcelUtils.write(fileName, list, "待入库数据")
|
||||
}
|
||||
|
||||
+210
@@ -0,0 +1,210 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
/**
|
||||
* 区域标准化
|
||||
* 1.小区库区域标准化,包含小区,商铺,写字楼
|
||||
* 2.poi数据库中 区域标准化
|
||||
*/
|
||||
class RegionStandardBatch extends BaseBatch{
|
||||
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
|
||||
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
|
||||
|
||||
static main(args) {
|
||||
runBatch(new RegionStandardBatch(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
// seaweedRegionStandard()
|
||||
poiRegionStandard()
|
||||
}
|
||||
|
||||
def seaweedRegionStandard(){
|
||||
seaweedRegionStandardByCat("shop")
|
||||
}
|
||||
def poiRegionStandard(){
|
||||
def citys = ["三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德"]//CommonDatautils.findAllCity()
|
||||
for(def city : citys){
|
||||
|
||||
def data = fullCityMap.get(city)
|
||||
def fullCityName = null
|
||||
if(data){
|
||||
fullCityName = data.city
|
||||
}
|
||||
if(fullCityName){
|
||||
|
||||
def rMap = regionFullNameMap.get(fullCityName)
|
||||
if(rMap){
|
||||
def set = rMap.keySet()
|
||||
def rets = findEsPoiByCity(fullCityName)
|
||||
println city +" "+rets.size()
|
||||
if(set.size()>1){
|
||||
poiHaveRegionProcess(city, set, rMap, rets)
|
||||
}else{
|
||||
// haveStreetProcess(city, set, cat)
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println "没找到相应的全称:"+city
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
def seaweedRegionStandardByCat(cat){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
for(def city : citys){
|
||||
|
||||
def rMap = regionMap.get(city)
|
||||
if(rMap){
|
||||
def set = rMap.keySet()
|
||||
if(set.size()>1){
|
||||
haveRegionProcess(city, set, rMap, cat)
|
||||
}else{
|
||||
haveStreetProcess(city, set, cat)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def haveRegionProcess(city, set, rMap, cat){
|
||||
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name"])
|
||||
for(def d : districts){
|
||||
if(set.contains(d.region)){
|
||||
continue
|
||||
}
|
||||
def updateRegion = null
|
||||
for(def key : set){
|
||||
def alias = rMap.get(key)
|
||||
if(alias.contains(d.region)){
|
||||
updateRegion = key
|
||||
break;
|
||||
}
|
||||
}
|
||||
if(updateRegion){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("region", updateRegion)
|
||||
udata.put("m_date", new Date())
|
||||
println d.city +" "+d.region+" "+d.name +" 区域修改为:"+updateRegion
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("status", 1)
|
||||
udata.put("m_date", new Date())
|
||||
println d.city +" "+d.region+" "+d.name +" 状态修改为:删除"
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def haveStreetProcess(city, set, cat){
|
||||
def data = fullCityMap.get(city)
|
||||
def fullCityName = null
|
||||
if(data){
|
||||
fullCityName = data.city
|
||||
}
|
||||
def streetMap = CommonDatautils.queryAllStreetNameByCity(fullCityName)
|
||||
if(streetMap){
|
||||
set = streetMap.keySet()
|
||||
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name"])
|
||||
for(def d : districts){
|
||||
if(set.contains(d.region)){
|
||||
continue
|
||||
}
|
||||
|
||||
def updateRegion = null
|
||||
def flag = null
|
||||
for(def key : set){
|
||||
def alias = streetMap.get(key)
|
||||
if((alias && alias.contains(d.region)) || (!alias && key.contains(d.region))){
|
||||
def newRegion = RegionData.formatRegion(d.city, key)
|
||||
if(newRegion!=d.region){
|
||||
updateRegion = newRegion
|
||||
}
|
||||
flag = "pass"
|
||||
break;
|
||||
}
|
||||
}
|
||||
if(updateRegion){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("region", updateRegion)
|
||||
udata.put("m_date", new Date())
|
||||
println d.city +" "+d.region+" "+d.name +" 区域修改为:"+updateRegion
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
if(flag!="pass"){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("status", 1)
|
||||
udata.put("m_date", new Date())
|
||||
println d.city +" "+d.region+" "+d.name +" 状态修改为:删除"
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def findEsPoiByCity(city){
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
|
||||
def rets = []
|
||||
|
||||
def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi_shards", queryBuilder)
|
||||
for(def d: datas){
|
||||
def o = new BasicDBObject()
|
||||
o.put("city", d.city)
|
||||
o.put("district", d.district)
|
||||
o.put("name", d.name)
|
||||
o.put("township", d.township)
|
||||
o.put("location", d.location)
|
||||
rets << o
|
||||
})
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
def poiHaveRegionProcess(city, set, rMap, datas){
|
||||
for(def d : datas){
|
||||
if(set.contains(d.region)){
|
||||
continue
|
||||
}
|
||||
def updateRegion = null
|
||||
for(def key : set){
|
||||
def alias = rMap.get(key)
|
||||
if(alias.contains(d.region)){
|
||||
updateRegion = key
|
||||
break;
|
||||
}
|
||||
}
|
||||
if(updateRegion){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("region", updateRegion)
|
||||
udata.put("m_date", new Date())
|
||||
println d.city +" "+d.region+" "+d.name +" 区域修改为:"+updateRegion
|
||||
|
||||
}else{
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("status", 1)
|
||||
udata.put("m_date", new Date())
|
||||
println d.city +" "+d.region+" "+d.name +" 状态修改为:删除"
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
+41
-9
@@ -22,53 +22,85 @@ class TownCenterBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def shapes = findHasShapeTown()
|
||||
def centers = findAllCenters()
|
||||
checkCenter(shapes, centers)
|
||||
def city = "上海市"
|
||||
def shapes = findHasShapeTown(city)
|
||||
def centers = findAllCenters(city)
|
||||
def retMap = checkCenter(shapes, centers)
|
||||
retMap.each {key,values->
|
||||
if(values.size()>1){
|
||||
println values
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def checkCenter(shapes, centers){
|
||||
def map = [:]
|
||||
def index = 0
|
||||
for(def town : shapes){
|
||||
index ++
|
||||
def points = InsidePolygon.createPoint4(town.shape.coordinates)
|
||||
//1.先判断当前镇的中心点是否在围栏中
|
||||
if(town.location){
|
||||
def flag = InsidePolygon.isInArea(town.location.lng, town.location.lat, points)
|
||||
def flag = InsidePolygon.isInArea(town.location.lon, town.location.lat, points)
|
||||
if(!flag){
|
||||
println town.city + " "+town.district+" "+town.street+" 不在本身围栏中"
|
||||
}else{
|
||||
def list = map.get(town.city + "#"+town.district+"#"+town.street)
|
||||
if(!list){
|
||||
list = []
|
||||
map.put(town.city + "#"+town.district+"#"+town.street, list)
|
||||
}
|
||||
list << town.city + "#"+town.district+"#"+town.street
|
||||
}
|
||||
}
|
||||
//2.判断是否有其它点在围栏中
|
||||
for(def center : centers){
|
||||
if(center._id.toString() != town._id.toString()){
|
||||
def flag = InsidePolygon.isInArea(center.location.lng, center.location.lat, points)
|
||||
def flag = InsidePolygon.isInArea(center.location.lon, center.location.lat, points)
|
||||
if(flag){
|
||||
def list = map.get(town.city + "#"+town.district+"#"+town.street)
|
||||
if(!list){
|
||||
list = []
|
||||
map.put(town.city + "#"+town.district+"#"+town.street)
|
||||
map.put(town.city + "#"+town.district+"#"+town.street, list)
|
||||
}
|
||||
list << center.city + "#"+center.district+"#"+center.street
|
||||
println center.city + " "+center.district+" "+center.street+" 在 "+town.city + " "+town.district+" "+town.street+" 围栏中"
|
||||
}
|
||||
}
|
||||
}
|
||||
def list = map.get(town.city + "#"+town.district+"#"+town.street)
|
||||
if(list){
|
||||
println index+" "+town.city + "#"+town.district+"#"+town.street + " "+list.size()
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
def findHasShapeTown(){
|
||||
def findHasShapeTown(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("type", "乡镇街道")
|
||||
q.put("shape", new BasicDBObject('$exists', true))
|
||||
return ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(2000).toList()
|
||||
if(city){
|
||||
q.put("city", city)
|
||||
}
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("location", 1)
|
||||
keys.put("city", 1)
|
||||
keys.put("district", 1)
|
||||
keys.put("street", 1)
|
||||
keys.put("shape", 1)
|
||||
return ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).toList()
|
||||
}
|
||||
|
||||
def findAllCenters(){
|
||||
def findAllCenters(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("type", "乡镇街道")
|
||||
q.put("location", new BasicDBObject('$exists', true))
|
||||
if(city){
|
||||
q.put("city", city)
|
||||
}
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("location", 1)
|
||||
keys.put("city", 1)
|
||||
|
||||
+54
-1
@@ -78,7 +78,7 @@ class TownMiddleProcess {
|
||||
//对比中间表的街道与community街道是否一致,因为元极反馈类似“高密市醴泉街道”解析有问题,要去掉“高密市”,然后去了中间表
|
||||
// updateCommunityDiffStreet()
|
||||
// updateTownPoline()//更新乡镇街道围栏
|
||||
updateTownPolineBatch()//批量更新乡镇街道围栏
|
||||
// updateTownPolineBatch()//批量更新乡镇街道围栏
|
||||
// checkTown()
|
||||
//区域民政部名称更新到乡镇街道中
|
||||
// updateTownshipByRegion()
|
||||
@@ -94,6 +94,8 @@ class TownMiddleProcess {
|
||||
|
||||
// addTownCenter()//百度接口添加中心点
|
||||
// checkCenterInTownRang()//检测乡镇街道中心点是否在围栏中
|
||||
|
||||
addSpecialCityTownAlias()
|
||||
}
|
||||
|
||||
|
||||
@@ -1301,6 +1303,57 @@ class TownMiddleProcess {
|
||||
for(def d : dbcursor){
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def addSpecialCityTownAlias(){
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
|
||||
def citys = ["东莞","中山","嘉峪关","昆玉","石河子","阿拉尔","图木舒克","五家渠","济源","仙桃","潜江","天门","神农架","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中","北屯","铁门关","双河","可克达拉"]
|
||||
def regionMap = CommonDatautils.getAllCityFullRegions()
|
||||
for(def city : citys){
|
||||
def townMap = regionMap.get(city)
|
||||
def streets = findAllStreetByCity(city)
|
||||
for(def s : streets){
|
||||
def set = townMap.keySet()
|
||||
def alias = null
|
||||
for(def name : set){
|
||||
def townObj = townMap.get(name)
|
||||
if(townObj && townObj.pattern && townObj.pattern.contains(s.street)){
|
||||
alias = townObj.pattern
|
||||
break;
|
||||
}
|
||||
}
|
||||
if(alias){
|
||||
def newAlias = []
|
||||
for(def n : alias){
|
||||
if(n!=s.street){
|
||||
newAlias << n
|
||||
}
|
||||
}
|
||||
if(alias){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("alias", newAlias)
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id", s._id), new BasicDBObject('$set', udata),false, false)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def findAllStreetByCity(city){
|
||||
def data = fullCityMap.get(city)
|
||||
def fullCityName = null
|
||||
if(data){
|
||||
fullCityName = data.city
|
||||
}
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("type", "乡镇街道")
|
||||
q.put("city", fullCityName)
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("street", 1)
|
||||
return ADMINISTRATIVE_MAPPING_COLL.find(q, keys).toList()
|
||||
}
|
||||
}
|
||||
|
||||
+12
-12
@@ -42,7 +42,7 @@ class TownshipStrandUpdate {
|
||||
// task.updateTown()
|
||||
// task.stat()
|
||||
|
||||
// task.updateDiffRegionTown()
|
||||
task.updateDiffRegionTown()
|
||||
// task.cleanTown()
|
||||
// task.checkTownFromMongo()
|
||||
// task.updateTownMapping()
|
||||
@@ -423,29 +423,29 @@ class TownshipStrandUpdate {
|
||||
}
|
||||
|
||||
def updateDiffRegionTown(){
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\shop-不同区域.xlsx"
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
def lists = CommonDatautils.findDistrictList(d['B'], null, d['D'], "shop")
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\district-不同区域.xlsx"
|
||||
ExcelUtils.modify(fileName,{d->
|
||||
def lists = CommonDatautils.findDistrictList(d['A'], null, d['C'], "district")
|
||||
d['AA'] = lists.size()
|
||||
//只处理库中只有一个的
|
||||
if(lists && lists.size()==1 && d['Z']=="包含" && !d['AB']){
|
||||
def newRegion = RegionData.formatRegion(d['B'], d['M'])
|
||||
if(lists && lists.size()==1 && d['R']){
|
||||
def newRegion = RegionData.formatRegion(d['A'], d['R'])
|
||||
if(newRegion){
|
||||
if(d['X'] && d['X'].toString().contains(d['M']+" "+d['N'])){ //地图街道在中间表中
|
||||
// if(d['X'] && d['X'].toString().contains(d['M']+" "+d['N'])){ //地图街道在中间表中
|
||||
def dbo = lists[0]
|
||||
def arrs = d['X'].split(" ")
|
||||
// def arrs = d['X'].split(" ")
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("region", newRegion)
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("township", arrs[1])
|
||||
// udata.put("township", arrs[1])
|
||||
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata))
|
||||
CommonDatautils.saveOptLog(dbo.city, dbo.region, dbo.name, "shop", "update","高娟")
|
||||
CommonDatautils.saveOptLog(dbo.city, dbo.region, dbo.name, "district", "update-region","高娟")
|
||||
d['AB'] = "已处理"
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
})
|
||||
},1)
|
||||
}
|
||||
|
||||
def cleanTown(){
|
||||
|
||||
+14
-9
@@ -2,6 +2,7 @@ package com.fangjia.script.gj.poc
|
||||
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
@@ -37,9 +38,9 @@ class AddPriceTest {
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def filePath = "D:\\数据\\建行\\房产估值\\建行业务反馈badcase.xlsx"
|
||||
// addGXDPrice(filePath)
|
||||
addCcbPrice(filePath)
|
||||
def filePath = "D:\\数据\\建行\\房产估值\\202410测试偏差较大案例统计数据test.xlsx"
|
||||
addGXDPrice(filePath)
|
||||
// addCcbPrice(filePath)
|
||||
// addThirdPrice(filePath)
|
||||
// addFjwPrice(filePath)
|
||||
// exportPkPrice()
|
||||
@@ -269,12 +270,16 @@ class AddPriceTest {
|
||||
|
||||
def addGXDPrice(filepath){
|
||||
ExcelUtils.multiThreadingModify(filepath, { d ->
|
||||
if (!d['M']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['B'])
|
||||
if (!d['B']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
if(dist){
|
||||
def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
|
||||
// def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
|
||||
d['M'] = gxdPrice
|
||||
if(gxdPrice){
|
||||
d['B'] = gxdPrice.price
|
||||
d['C'] = DateUtils.formatDateByPattern(gxdPrice.c_date, "yyyy-MM-dd")
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -288,11 +293,11 @@ class AddPriceTest {
|
||||
q.put("district_name", name)
|
||||
q.put("source", "GXD")
|
||||
def dbcursor = confirm_price_date.find(q).sort(new BasicDBObject("c_date", -1)).limit(1)
|
||||
def price = null
|
||||
def obj = null
|
||||
for(def d : dbcursor){
|
||||
price = d.price
|
||||
obj = d
|
||||
}
|
||||
return price
|
||||
return obj
|
||||
}
|
||||
|
||||
|
||||
|
||||
+9
-8
@@ -236,6 +236,7 @@ class ElasticUtils89 {
|
||||
searchRequest.scroll(scroll);
|
||||
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||
searchSourceBuilder.query(qb);
|
||||
searchSourceBuilder.size(100)
|
||||
searchRequest.source(searchSourceBuilder);
|
||||
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
|
||||
String scrollId = searchResponse.getScrollId();
|
||||
@@ -247,14 +248,14 @@ class ElasticUtils89 {
|
||||
searchHit.getSourceAsMap().put("id", searchHit.getId());
|
||||
results << searchHit.getSourceAsMap()
|
||||
}
|
||||
if(results.size()%100000==0){
|
||||
println "."+ new Date()
|
||||
}
|
||||
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
|
||||
scrollRequest.scroll(scroll);
|
||||
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
|
||||
scrollId = searchResponse.getScrollId();
|
||||
searchHits = searchResponse.getHits().getHits();
|
||||
// if(results.size()%100000==0){
|
||||
// println "."+ new Date()
|
||||
// }
|
||||
// SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
|
||||
// scrollRequest.scroll(scroll);
|
||||
// searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
|
||||
// scrollId = searchResponse.getScrollId();
|
||||
// searchHits = searchResponse.getHits().getHits();
|
||||
}
|
||||
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
|
||||
clearScrollRequest.addScrollId(scrollId);
|
||||
|
||||
+4
-4
@@ -150,8 +150,8 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def dist = dists[i]
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
|
||||
@@ -171,8 +171,8 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
// }
|
||||
// });
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
|
||||
@@ -414,7 +414,7 @@ public class ExcelUtils{
|
||||
|
||||
def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
|
||||
def body = data.subList(1, data.size())
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 4
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 6
|
||||
println nThread
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
|
||||
final AtomicInteger counter = new AtomicInteger(0)
|
||||
|
||||
Reference in New Issue
Block a user