Merge branch 'script_gj' into 'master'

update

See merge request !985
This commit is contained in:
2024-10-29 09:47:43 +08:00
12 changed files with 552 additions and 95 deletions
@@ -1,11 +1,18 @@
package com.fangjia.script.ccbevaluation package com.fangjia.script.ccbevaluation
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/** /**
* 将国信达及成交对标数据加入到第三方均价表,做为均价参考 * 将国信达及成交对标数据加入到第三方均价表,做为均价参考
*/ */
@@ -17,18 +24,24 @@ class AddThirdPriceSample {
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
static main(args) { static main(args) {
def task = new AddThirdPriceSample() def task = new AddThirdPriceSample()
task.run() //添加第三方 fccs->保存的是国信达的参考价格 csjj->保存的是国信达的成交数据 // task.run() //添加第三方 fccs->保存的是国信达的参考价格 csjj->保存的是国信达的成交数据
// task.addTrade() //添加成交 task.addTrade() //添加成交
// task.matchOldData()
// task.noInCcbCity()
} }
def run(){ def run(){
for(def i=0;i<19;i++){ for(def i=0;i<1;i++){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\国信达参考价作为第三方入库.xlsx",{d-> ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\国信达指导价录入-1024.xlsx",{d->
if(d['A'] && d['B']){ if(d['A'] && d['B']){
def dist = CommonDatautils.findDistrictById(d['A'].toString().trim()) def dist = CommonDatautils.findDistrictById(d['A'].toString().trim())
if(dist){ if(dist){
def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202408) def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202409)
// def udata = new BasicDBObject("csjj_esf_price", d['C'].toString().toDouble().toInteger()) // def udata = new BasicDBObject("csjj_esf_price", d['C'].toString().toDouble().toInteger())
def udata = new BasicDBObject("fccs_esf_price", d['B'].toString().toDouble().toInteger()) def udata = new BasicDBObject("fccs_esf_price", d['B'].toString().toDouble().toInteger())
AVERAGE_STATISTICS.update(q,new BasicDBObject('$set', udata),false, false) AVERAGE_STATISTICS.update(q,new BasicDBObject('$set', udata),false, false)
@@ -44,15 +57,16 @@ class AddThirdPriceSample {
} }
def addTrade(){ def addTrade(){
def citys = ["武汉","南京","杭州","长沙","郑州","合肥","宁波","南通","嘉兴","珠海","泉州","绍兴","湖州","镇江"] // def citys = ["武汉","南京","杭州","长沙","郑州","合肥","宁波","南通","嘉兴","珠海","泉州","绍兴","湖州","镇江"]
def rets = findTrades(citys) def rets = findTrades()
// DEAL_PRICE_COLL.insert(rets) DEAL_PRICE_COLL.insert(rets)
} }
def findTrades(citys){ def findTrades(){
def q = new BasicDBObject() def q = new BasicDBObject()
q.put("city", new BasicDBObject('$in', citys)) q.put("ccb_price_remark","建行无此城市成交")
q.put("flag", 1) // q.put("city", new BasicDBObject('$in', citys))
// q.put("flag", 1)
def dbcursor = NO_PRICE_DEAL_PRICE_COLL.find(q) def dbcursor = NO_PRICE_DEAL_PRICE_COLL.find(q)
def rets = [] def rets = []
def index = 0 def index = 0
@@ -62,7 +76,7 @@ class AddThirdPriceSample {
udata.put("_id",db._id) udata.put("_id",db._id)
udata.put("url",db.url) udata.put("url",db.url)
udata.put("area",db.area) udata.put("area",db.area.toString().toDouble())
udata.put("avg_price",db.avg_price.toString().toDouble().toInteger()) udata.put("avg_price",db.avg_price.toString().toDouble().toInteger())
udata.put("build_structure",db.build_structure) udata.put("build_structure",db.build_structure)
udata.put("build_type",db.build_type) udata.put("build_type",db.build_type)
@@ -73,8 +87,14 @@ class AddThirdPriceSample {
udata.put("elevator",db.elevator) udata.put("elevator",db.elevator)
udata.put("fitment",db.fitment) udata.put("fitment",db.fitment)
udata.put("floor_info",db.floor_info) udata.put("floor_info",db.floor_info)
udata.put("hall",db.hall.toString().toInteger()) if(db.hall){
udata.put("height",db.height.toString().toInteger()) udata.put("hall",db.hall.toString().toInteger())
}
if(db.height){
udata.put("height",db.height.toString().toInteger())
}
udata.put("housing_age",db.housing_age) udata.put("housing_age",db.housing_age)
udata.put("lift",db.lift.toString().toInteger()) udata.put("lift",db.lift.toString().toInteger())
udata.put("lift_household",db.lift_household.toString().toInteger()) udata.put("lift_household",db.lift_household.toString().toInteger())
@@ -83,7 +103,7 @@ class AddThirdPriceSample {
udata.put("room", db.room.toString().toInteger()) udata.put("room", db.room.toString().toInteger())
udata.put("room_structure",db.room_structure) udata.put("room_structure",db.room_structure)
udata.put("source",db.source) udata.put("source",db.source)
udata.put("total_price",db.total_price.toString().toInteger()) udata.put("total_price",db.total_price.toString().toDouble().toInteger())
udata.put("trade_date",db.trade_date) udata.put("trade_date",db.trade_date)
udata.put("trade_property",db.trade_property) udata.put("trade_property",db.trade_property)
udata.put("house_using",db.house_using) udata.put("house_using",db.house_using)
@@ -98,11 +118,79 @@ class AddThirdPriceSample {
udata.put("match_id",db.match_id) udata.put("match_id",db.match_id)
udata.put("price_flag", "国信达成交-推导") udata.put("price_flag", "国信达成交-推导")
DEAL_PRICE_COLL.save(udata) // DEAL_PRICE_COLL.save(udata)
println index println index
// rets << udata rets << udata
} }
return rets return rets
} }
def matchOldData(){
def q = new BasicDBObject("source","贝壳找房").append("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2024-09-01","yyyy-MM-dd")))
def list = DEAL_PRICE_COLL.find(q).toList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(list.size())
for(def i=0;i<list.size();i++){
final def d = list[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def query = new BasicDBObject("url", d.url)
def obj = NO_PRICE_DEAL_PRICE_COLL.findOne(query)
if(obj){
def update = new BasicDBObject()
update.put("price_remark", "有价格")
NO_PRICE_DEAL_PRICE_COLL.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set', update), false,false)
}
println totalCount.get()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
def noInCcbCity(){
def citys = ["大理","苏州","无锡","通辽","绵阳","吉安","潍坊","襄阳","济宁","张家口","泸州","乐山","赣州","九江","眉山","桂林","雅安","广州","景德镇","上海","运城","内江","金华","濮阳","丹东","淄博","新乡","廊坊","烟台","邯郸","萍乡","宜宾","咸阳","许昌","德阳","银川","洛阳","天津","湛江","长沙","深圳","湘西","信阳","成都","威海","福州","上饶","常州","宜春","天水","平顶山","常德","吉林","盐城","杭州","赤峰","南京","周口","济南","株洲","大连","徐州","泰安","太原","凉山","兰州","承德","广元","台州","南昌","乌鲁木齐","海口","北京","淮安","清远","临沂","柳州","衢州","攀枝花","呼和浩特","昆明","岳阳","衡阳","温州","石家庄","连云港","保定","阜阳","遂宁","菏泽","青岛","唐山","开封","漳州","贵阳","驻马店","宝鸡","达州","汉中","南充","遵义","包头","南宁","沈阳","重庆","芜湖","厦门","大同"]
def q = new BasicDBObject()
q.put("price_remark",new BasicDBObject('$ne', "有价格"))
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2024-09-01","yyyy-MM-dd")))
q.put("avg_price",new BasicDBObject('$gt', 0))
q.put("match_id",new BasicDBObject('$exists', true))
def list = NO_PRICE_DEAL_PRICE_COLL.find(q).toList()
println "共:"+list.size()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(list.size())
for(def i=0;i<list.size();i++){
final def d = list[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
if(!citys.contains(d.city)){
def update = new BasicDBObject()
update.put("ccb_price_remark", "建行无此城市成交")
NO_PRICE_DEAL_PRICE_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set', update), false,false)
}
println totalCount.get()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
} }
@@ -1234,15 +1234,7 @@ def findThirdCommCount(cityid, sqlInstance){
//} //}
// //
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\无价格小区(3)2.23.xlsx"
//ExcelUtils.multiThreadingModify(fileName, { d ->
// def dist = CommonDatautils.findDistrictById(d['A'])
// if(dist){
// d['G'] = dist.city
// d['H'] = dist.region
// d['I'] = dist.name
// }
//})
@@ -1331,34 +1323,45 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
def fileName = "D:\\数据\\建行\\房产估值\\6个城市待优化_9月xlsx.xlsx" //def fileName = "D:\\数据\\建行\\房产估值\\6个城市待优化_9月xlsx.xlsx"
// ////
ExcelUtils.multiThreadingModify(fileName, {d-> //ExcelUtils.multiThreadingModify(fileName, {d->
if(d['G']){ // if(d['G']){
// def region = RegionData.formatRegion(d['A'], d['J']) //// def region = RegionData.formatRegion(d['A'], d['J'])
// if(region){ //// if(region){
// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district") //// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district")
//// if(dist){
//// d['B'] = dist.region
//// d['C'] = dist.name
//// d['L'] = dist.status-dist.name_cat
//// }
//// }
// if(d['G'] instanceof String){
// def dist = CommonDatautils.findDistrictById(d['G'])
// if(dist){ // if(dist){
// d['B'] = dist.region //// d['A'] = dist.city
// d['C'] = dist.name //// d['B'] = dist.region
// d['L'] = dist.status-dist.name_cat //// d['C'] = dist.name
//// d['K'] = dist.estate_type2
//// d['L'] = dist.status
//// d['M'] = dist.name_cat
//
// d['H'] = dist.name
// } // }
// } // }
if(d['G'] instanceof String){ // }
def dist = CommonDatautils.findDistrictById(d['G']) //},0)
if(dist){
// d['A'] = dist.city
// d['B'] = dist.region
// d['C'] = dist.name
// d['K'] = dist.estate_type2
// d['L'] = dist.status
// d['M'] = dist.name_cat
d['H'] = dist.name
} def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20241025人工指导价.xlsx"
} ExcelUtils.multiThreadingModify(fileName, { d ->
def dist = CommonDatautils.findDistrictById(d['J'])
if(dist){
d['A'] = dist.city
d['B'] = dist.region
d['C'] = dist.name
} }
},0) })
//def file = "D:\\数据\\建行\\房产估值\\指导价\\5城价格翻新6-21.xlsx" //def file = "D:\\数据\\建行\\房产估值\\指导价\\5城价格翻新6-21.xlsx"
@@ -4785,7 +4785,6 @@ class CommonDatautils {
def q = new BasicDBObject() def q = new BasicDBObject()
q.put("type","区域") q.put("type","区域")
q.put("district_diff",1) q.put("district_diff",1)
q.put("city",new BasicDBObject('$in', ["深圳市"]))
def keys = new BasicDBObject() def keys = new BasicDBObject()
keys.put("province",1) keys.put("province",1)
@@ -4814,6 +4813,72 @@ class CommonDatautils {
return regionMap return regionMap
} }
static def queryAllRegionNameByCity(){
def q = new BasicDBObject()
q.put("type","区域")
q.put("district_diff",1)
def keys = new BasicDBObject()
keys.put("city_short",1)
keys.put("district_short",1)
keys.put("alias",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def retMap = [:]
for(def d : dbcursor){
def regionMap = retMap.get(d.city_short)
if(!regionMap){
regionMap = [:]
retMap.put(d.city_short, regionMap)
}
regionMap.put(d.district_short, d.alias)
}
return retMap
}
static def queryAllRegionFullNameByCity(){
def q = new BasicDBObject()
q.put("type","区域")
q.put("district_diff",1)
def keys = new BasicDBObject()
keys.put("city",1)
keys.put("district",1)
keys.put("alias",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def retMap = [:]
for(def d : dbcursor){
def regionMap = retMap.get(d.city)
if(!regionMap){
regionMap = [:]
retMap.put(d.city, regionMap)
}
regionMap.put(d.district, d.alias)
}
return retMap
}
static def queryAllStreetNameByCity(city){
def q = new BasicDBObject()
q.put("type","乡镇街道")
q.put("city", city)
def keys = new BasicDBObject()
keys.put("street",1)
keys.put("alias",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def retMap = [:]
for(def d : dbcursor){
retMap.put(d.street, d.alias)
}
return retMap
}
static def queryCommunityByCity(){ static def queryCommunityByCity(){
def q = new BasicDBObject() def q = new BasicDBObject()
@@ -49,7 +49,7 @@ class AutoAddDistrictBatch extends BaseBatch{
def fileName = "/home/goojia/gj/add_dist_info.xlsx" def fileName = "/home/goojia/gj/add_dist_info.xlsx"
if (CommonDatautils.isWindow()) { if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\名录集\\第三方待入库数据.xlsx" fileName = "D:\\数据\\数据建设\\小区\\第三方待入库数据.xlsx"
} }
ExcelUtils.write(fileName, list, "待入库数据") ExcelUtils.write(fileName, list, "待入库数据")
} }
@@ -0,0 +1,210 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
/**
* 区域标准化
* 1.小区库区域标准化,包含小区,商铺,写字楼
* 2.poi数据库中 区域标准化
*/
class RegionStandardBatch extends BaseBatch{
def fullCityMap = CommonDatautils.queryFullNameCity()
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
static main(args) {
runBatch(new RegionStandardBatch(), args)
}
@Override
void process(Object args) {
// seaweedRegionStandard()
poiRegionStandard()
}
def seaweedRegionStandard(){
seaweedRegionStandardByCat("shop")
}
def poiRegionStandard(){
def citys = ["三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德"]//CommonDatautils.findAllCity()
for(def city : citys){
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
fullCityName = data.city
}
if(fullCityName){
def rMap = regionFullNameMap.get(fullCityName)
if(rMap){
def set = rMap.keySet()
def rets = findEsPoiByCity(fullCityName)
println city +" "+rets.size()
if(set.size()>1){
poiHaveRegionProcess(city, set, rMap, rets)
}else{
// haveStreetProcess(city, set, cat)
}
}
}else{
println "没找到相应的全称:"+city
}
}
}
def seaweedRegionStandardByCat(cat){
def citys = CommonDatautils.findAllCity()
for(def city : citys){
def rMap = regionMap.get(city)
if(rMap){
def set = rMap.keySet()
if(set.size()>1){
haveRegionProcess(city, set, rMap, cat)
}else{
haveStreetProcess(city, set, cat)
}
}
}
}
def haveRegionProcess(city, set, rMap, cat){
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name"])
for(def d : districts){
if(set.contains(d.region)){
continue
}
def updateRegion = null
for(def key : set){
def alias = rMap.get(key)
if(alias.contains(d.region)){
updateRegion = key
break;
}
}
if(updateRegion){
def udata = new BasicDBObject()
udata.put("region", updateRegion)
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name +" 区域修改为:"+updateRegion
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
}else{
def udata = new BasicDBObject()
udata.put("status", 1)
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name +" 状态修改为:删除"
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
}
}
}
def haveStreetProcess(city, set, cat){
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
fullCityName = data.city
}
def streetMap = CommonDatautils.queryAllStreetNameByCity(fullCityName)
if(streetMap){
set = streetMap.keySet()
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name"])
for(def d : districts){
if(set.contains(d.region)){
continue
}
def updateRegion = null
def flag = null
for(def key : set){
def alias = streetMap.get(key)
if((alias && alias.contains(d.region)) || (!alias && key.contains(d.region))){
def newRegion = RegionData.formatRegion(d.city, key)
if(newRegion!=d.region){
updateRegion = newRegion
}
flag = "pass"
break;
}
}
if(updateRegion){
def udata = new BasicDBObject()
udata.put("region", updateRegion)
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name +" 区域修改为:"+updateRegion
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
}else{
if(flag!="pass"){
def udata = new BasicDBObject()
udata.put("status", 1)
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name +" 状态修改为:删除"
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
}
}
}
}
}
def findEsPoiByCity(city){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
def rets = []
def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi_shards", queryBuilder)
for(def d: datas){
def o = new BasicDBObject()
o.put("city", d.city)
o.put("district", d.district)
o.put("name", d.name)
o.put("township", d.township)
o.put("location", d.location)
rets << o
})
return rets
}
def poiHaveRegionProcess(city, set, rMap, datas){
for(def d : datas){
if(set.contains(d.region)){
continue
}
def updateRegion = null
for(def key : set){
def alias = rMap.get(key)
if(alias.contains(d.region)){
updateRegion = key
break;
}
}
if(updateRegion){
def udata = new BasicDBObject()
udata.put("region", updateRegion)
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name +" 区域修改为:"+updateRegion
}else{
def udata = new BasicDBObject()
udata.put("status", 1)
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name +" 状态修改为:删除"
}
}
}
}
@@ -22,53 +22,85 @@ class TownCenterBatch extends BaseBatch{
@Override @Override
void process(Object args) { void process(Object args) {
def shapes = findHasShapeTown() def city = "上海市"
def centers = findAllCenters() def shapes = findHasShapeTown(city)
checkCenter(shapes, centers) def centers = findAllCenters(city)
def retMap = checkCenter(shapes, centers)
retMap.each {key,values->
if(values.size()>1){
println values
}
}
} }
def checkCenter(shapes, centers){ def checkCenter(shapes, centers){
def map = [:] def map = [:]
def index = 0
for(def town : shapes){ for(def town : shapes){
index ++
def points = InsidePolygon.createPoint4(town.shape.coordinates) def points = InsidePolygon.createPoint4(town.shape.coordinates)
//1.先判断当前镇的中心点是否在围栏中 //1.先判断当前镇的中心点是否在围栏中
if(town.location){ if(town.location){
def flag = InsidePolygon.isInArea(town.location.lng, town.location.lat, points) def flag = InsidePolygon.isInArea(town.location.lon, town.location.lat, points)
if(!flag){ if(!flag){
println town.city + " "+town.district+" "+town.street+" 不在本身围栏中" println town.city + " "+town.district+" "+town.street+" 不在本身围栏中"
}else{
def list = map.get(town.city + "#"+town.district+"#"+town.street)
if(!list){
list = []
map.put(town.city + "#"+town.district+"#"+town.street, list)
}
list << town.city + "#"+town.district+"#"+town.street
} }
} }
//2.判断是否有其它点在围栏中 //2.判断是否有其它点在围栏中
for(def center : centers){ for(def center : centers){
if(center._id.toString() != town._id.toString()){ if(center._id.toString() != town._id.toString()){
def flag = InsidePolygon.isInArea(center.location.lng, center.location.lat, points) def flag = InsidePolygon.isInArea(center.location.lon, center.location.lat, points)
if(flag){ if(flag){
def list = map.get(town.city + "#"+town.district+"#"+town.street) def list = map.get(town.city + "#"+town.district+"#"+town.street)
if(!list){ if(!list){
list = [] list = []
map.put(town.city + "#"+town.district+"#"+town.street) map.put(town.city + "#"+town.district+"#"+town.street, list)
} }
list << center.city + "#"+center.district+"#"+center.street list << center.city + "#"+center.district+"#"+center.street
println center.city + " "+center.district+" "+center.street+" 在 "+town.city + " "+town.district+" "+town.street+" 围栏中" println center.city + " "+center.district+" "+center.street+" 在 "+town.city + " "+town.district+" "+town.street+" 围栏中"
} }
} }
} }
def list = map.get(town.city + "#"+town.district+"#"+town.street)
if(list){
println index+" "+town.city + "#"+town.district+"#"+town.street + " "+list.size()
}
} }
return map return map
} }
def findHasShapeTown(){ def findHasShapeTown(city){
def q = new BasicDBObject() def q = new BasicDBObject()
q.put("type", "乡镇街道") q.put("type", "乡镇街道")
q.put("shape", new BasicDBObject('$exists', true)) q.put("shape", new BasicDBObject('$exists', true))
return ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(2000).toList() if(city){
q.put("city", city)
}
def keys = new BasicDBObject()
keys.put("location", 1)
keys.put("city", 1)
keys.put("district", 1)
keys.put("street", 1)
keys.put("shape", 1)
return ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).toList()
} }
def findAllCenters(){ def findAllCenters(city){
def q = new BasicDBObject() def q = new BasicDBObject()
q.put("type", "乡镇街道") q.put("type", "乡镇街道")
q.put("location", new BasicDBObject('$exists', true)) q.put("location", new BasicDBObject('$exists', true))
if(city){
q.put("city", city)
}
def keys = new BasicDBObject() def keys = new BasicDBObject()
keys.put("location", 1) keys.put("location", 1)
keys.put("city", 1) keys.put("city", 1)
@@ -78,7 +78,7 @@ class TownMiddleProcess {
//对比中间表的街道与community街道是否一致,因为元极反馈类似“高密市醴泉街道”解析有问题,要去掉“高密市”,然后去了中间表 //对比中间表的街道与community街道是否一致,因为元极反馈类似“高密市醴泉街道”解析有问题,要去掉“高密市”,然后去了中间表
// updateCommunityDiffStreet() // updateCommunityDiffStreet()
// updateTownPoline()//更新乡镇街道围栏 // updateTownPoline()//更新乡镇街道围栏
updateTownPolineBatch()//批量更新乡镇街道围栏 // updateTownPolineBatch()//批量更新乡镇街道围栏
// checkTown() // checkTown()
//区域民政部名称更新到乡镇街道中 //区域民政部名称更新到乡镇街道中
// updateTownshipByRegion() // updateTownshipByRegion()
@@ -94,6 +94,8 @@ class TownMiddleProcess {
// addTownCenter()//百度接口添加中心点 // addTownCenter()//百度接口添加中心点
// checkCenterInTownRang()//检测乡镇街道中心点是否在围栏中 // checkCenterInTownRang()//检测乡镇街道中心点是否在围栏中
addSpecialCityTownAlias()
} }
@@ -1301,6 +1303,57 @@ class TownMiddleProcess {
for(def d : dbcursor){ for(def d : dbcursor){
} }
}
def addSpecialCityTownAlias(){
def fullCityMap = CommonDatautils.queryFullNameCity()
def citys = ["东莞","中山","嘉峪关","昆玉","石河子","阿拉尔","图木舒克","五家渠","济源","仙桃","潜江","天门","神农架","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中","北屯","铁门关","双河","可克达拉"]
def regionMap = CommonDatautils.getAllCityFullRegions()
for(def city : citys){
def townMap = regionMap.get(city)
def streets = findAllStreetByCity(city)
for(def s : streets){
def set = townMap.keySet()
def alias = null
for(def name : set){
def townObj = townMap.get(name)
if(townObj && townObj.pattern && townObj.pattern.contains(s.street)){
alias = townObj.pattern
break;
}
}
if(alias){
def newAlias = []
for(def n : alias){
if(n!=s.street){
newAlias << n
}
}
if(alias){
def udata = new BasicDBObject()
udata.put("alias", newAlias)
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id", s._id), new BasicDBObject('$set', udata),false, false)
}
}
}
}
}
def findAllStreetByCity(city){
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
fullCityName = data.city
}
def q = new BasicDBObject()
q.put("type", "乡镇街道")
q.put("city", fullCityName)
def keys = new BasicDBObject()
keys.put("street", 1)
return ADMINISTRATIVE_MAPPING_COLL.find(q, keys).toList()
} }
} }
@@ -42,7 +42,7 @@ class TownshipStrandUpdate {
// task.updateTown() // task.updateTown()
// task.stat() // task.stat()
// task.updateDiffRegionTown() task.updateDiffRegionTown()
// task.cleanTown() // task.cleanTown()
// task.checkTownFromMongo() // task.checkTownFromMongo()
// task.updateTownMapping() // task.updateTownMapping()
@@ -423,29 +423,29 @@ class TownshipStrandUpdate {
} }
def updateDiffRegionTown(){ def updateDiffRegionTown(){
def fileName = "D:\\数据\\数据建设\\乡镇街道\\shop-不同区域.xlsx" def fileName = "D:\\数据\\数据建设\\乡镇街道\\district-不同区域.xlsx"
ExcelUtils.multiThreadingModify(fileName,{d-> ExcelUtils.modify(fileName,{d->
def lists = CommonDatautils.findDistrictList(d['B'], null, d['D'], "shop") def lists = CommonDatautils.findDistrictList(d['A'], null, d['C'], "district")
d['AA'] = lists.size() d['AA'] = lists.size()
//只处理库中只有一个的 //只处理库中只有一个的
if(lists && lists.size()==1 && d['Z']=="包含" && !d['AB']){ if(lists && lists.size()==1 && d['R']){
def newRegion = RegionData.formatRegion(d['B'], d['M']) def newRegion = RegionData.formatRegion(d['A'], d['R'])
if(newRegion){ if(newRegion){
if(d['X'] && d['X'].toString().contains(d['M']+" "+d['N'])){ //地图街道在中间表中 // if(d['X'] && d['X'].toString().contains(d['M']+" "+d['N'])){ //地图街道在中间表中
def dbo = lists[0] def dbo = lists[0]
def arrs = d['X'].split(" ") // def arrs = d['X'].split(" ")
def udata = new BasicDBObject() def udata = new BasicDBObject()
udata.put("region", newRegion) udata.put("region", newRegion)
udata.put("m_date", new Date()) udata.put("m_date", new Date())
udata.put("township", arrs[1]) // udata.put("township", arrs[1])
DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata)) DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata))
CommonDatautils.saveOptLog(dbo.city, dbo.region, dbo.name, "shop", "update","高娟") CommonDatautils.saveOptLog(dbo.city, dbo.region, dbo.name, "district", "update-region","高娟")
d['AB'] = "已处理" d['AB'] = "已处理"
} // }
} }
} }
}) },1)
} }
def cleanTown(){ def cleanTown(){
@@ -2,6 +2,7 @@ package com.fangjia.script.gj.poc
import com.fangjia.script.ccbevaluation.CcbCommonUtils import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtil
import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
@@ -37,9 +38,9 @@ class AddPriceTest {
task.run(); task.run();
} }
def run(){ def run(){
def filePath = "D:\\数据\\建行\\房产估值\\建行业务反馈badcase.xlsx" def filePath = "D:\\数据\\建行\\房产估值\\202410测试偏差较大案例统计数据test.xlsx"
// addGXDPrice(filePath) addGXDPrice(filePath)
addCcbPrice(filePath) // addCcbPrice(filePath)
// addThirdPrice(filePath) // addThirdPrice(filePath)
// addFjwPrice(filePath) // addFjwPrice(filePath)
// exportPkPrice() // exportPkPrice()
@@ -269,12 +270,16 @@ class AddPriceTest {
def addGXDPrice(filepath){ def addGXDPrice(filepath){
ExcelUtils.multiThreadingModify(filepath, { d -> ExcelUtils.multiThreadingModify(filepath, { d ->
if (!d['M']) { if (!d['B']) {
def dist = CommonDatautils.findDistrictById(d['B']) def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){ if(dist){
def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name) def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
// def gxdPrice = findGXDPrice(d['D'], d['E'], d['C']) // def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
d['M'] = gxdPrice if(gxdPrice){
d['B'] = gxdPrice.price
d['C'] = DateUtils.formatDateByPattern(gxdPrice.c_date, "yyyy-MM-dd")
}
} }
} }
}) })
@@ -288,11 +293,11 @@ class AddPriceTest {
q.put("district_name", name) q.put("district_name", name)
q.put("source", "GXD") q.put("source", "GXD")
def dbcursor = confirm_price_date.find(q).sort(new BasicDBObject("c_date", -1)).limit(1) def dbcursor = confirm_price_date.find(q).sort(new BasicDBObject("c_date", -1)).limit(1)
def price = null def obj = null
for(def d : dbcursor){ for(def d : dbcursor){
price = d.price obj = d
} }
return price return obj
} }
@@ -236,6 +236,7 @@ class ElasticUtils89 {
searchRequest.scroll(scroll); searchRequest.scroll(scroll);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(qb); searchSourceBuilder.query(qb);
searchSourceBuilder.size(100)
searchRequest.source(searchSourceBuilder); searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
String scrollId = searchResponse.getScrollId(); String scrollId = searchResponse.getScrollId();
@@ -247,14 +248,14 @@ class ElasticUtils89 {
searchHit.getSourceAsMap().put("id", searchHit.getId()); searchHit.getSourceAsMap().put("id", searchHit.getId());
results << searchHit.getSourceAsMap() results << searchHit.getSourceAsMap()
} }
if(results.size()%100000==0){ // if(results.size()%100000==0){
println "."+ new Date() // println "."+ new Date()
} // }
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); // SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll); // scrollRequest.scroll(scroll);
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT); // searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
scrollId = searchResponse.getScrollId(); // scrollId = searchResponse.getScrollId();
searchHits = searchResponse.getHits().getHits(); // searchHits = searchResponse.getHits().getHits();
} }
ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.addScrollId(scrollId); clearScrollRequest.addScrollId(scrollId);
@@ -150,8 +150,8 @@ class SeaweedDataSynch extends BaseBatch{
final CountDownLatch latch = new CountDownLatch(dists.size()) final CountDownLatch latch = new CountDownLatch(dists.size())
for(def i=0;i<dists.size();i++){ for(def i=0;i<dists.size();i++){
final def dist = dists[i] final def dist = dists[i]
// fixedThreadPool.execute(new Runnable() { fixedThreadPool.execute(new Runnable() {
// void run() { void run() {
try{ try{
totalCount.addAndGet(1) totalCount.addAndGet(1)
@@ -171,8 +171,8 @@ class SeaweedDataSynch extends BaseBatch{
} finally { } finally {
latch.countDown() latch.countDown()
} }
// } }
// }); });
} }
latch.await() latch.await()
return rets; return rets;
@@ -414,7 +414,7 @@ public class ExcelUtils{
def data = ExcelUtils.read(fileName, srcSheetIndex, 0) def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
def body = data.subList(1, data.size()) def body = data.subList(1, data.size())
def nThread = Runtime.getRuntime().availableProcessors() * 4 def nThread = Runtime.getRuntime().availableProcessors() * 6
println nThread println nThread
ExecutorService executorService = Executors.newFixedThreadPool(nThread) ExecutorService executorService = Executors.newFixedThreadPool(nThread)
final AtomicInteger counter = new AtomicInteger(0) final AtomicInteger counter = new AtomicInteger(0)