Merge branch 'script_gj' into 'master'

update

See merge request !985
This commit is contained in:
2024-10-29 09:47:43 +08:00
12 changed files with 552 additions and 95 deletions
@@ -1,11 +1,18 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 将国信达及成交对标数据加入到第三方均价表,做为均价参考
*/
@@ -17,18 +24,24 @@ class AddThirdPriceSample {
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
static main(args) {
def task = new AddThirdPriceSample()
task.run() //添加第三方 fccs->保存的是国信达的参考价格 csjj->保存的是国信达的成交数据
// task.addTrade() //添加成交
// task.run() //添加第三方 fccs->保存的是国信达的参考价格 csjj->保存的是国信达的成交数据
task.addTrade() //添加成交
// task.matchOldData()
// task.noInCcbCity()
}
def run(){
for(def i=0;i<19;i++){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\国信达参考价作为第三方入库.xlsx",{d->
for(def i=0;i<1;i++){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\国信达指导价录入-1024.xlsx",{d->
if(d['A'] && d['B']){
def dist = CommonDatautils.findDistrictById(d['A'].toString().trim())
if(dist){
def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202408)
def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202409)
// def udata = new BasicDBObject("csjj_esf_price", d['C'].toString().toDouble().toInteger())
def udata = new BasicDBObject("fccs_esf_price", d['B'].toString().toDouble().toInteger())
AVERAGE_STATISTICS.update(q,new BasicDBObject('$set', udata),false, false)
@@ -44,15 +57,16 @@ class AddThirdPriceSample {
}
def addTrade(){
def citys = ["武汉","南京","杭州","长沙","郑州","合肥","宁波","南通","嘉兴","珠海","泉州","绍兴","湖州","镇江"]
def rets = findTrades(citys)
// DEAL_PRICE_COLL.insert(rets)
// def citys = ["武汉","南京","杭州","长沙","郑州","合肥","宁波","南通","嘉兴","珠海","泉州","绍兴","湖州","镇江"]
def rets = findTrades()
DEAL_PRICE_COLL.insert(rets)
}
def findTrades(citys){
def findTrades(){
def q = new BasicDBObject()
q.put("city", new BasicDBObject('$in', citys))
q.put("flag", 1)
q.put("ccb_price_remark","建行无此城市成交")
// q.put("city", new BasicDBObject('$in', citys))
// q.put("flag", 1)
def dbcursor = NO_PRICE_DEAL_PRICE_COLL.find(q)
def rets = []
def index = 0
@@ -62,7 +76,7 @@ class AddThirdPriceSample {
udata.put("_id",db._id)
udata.put("url",db.url)
udata.put("area",db.area)
udata.put("area",db.area.toString().toDouble())
udata.put("avg_price",db.avg_price.toString().toDouble().toInteger())
udata.put("build_structure",db.build_structure)
udata.put("build_type",db.build_type)
@@ -73,8 +87,14 @@ class AddThirdPriceSample {
udata.put("elevator",db.elevator)
udata.put("fitment",db.fitment)
udata.put("floor_info",db.floor_info)
udata.put("hall",db.hall.toString().toInteger())
udata.put("height",db.height.toString().toInteger())
if(db.hall){
udata.put("hall",db.hall.toString().toInteger())
}
if(db.height){
udata.put("height",db.height.toString().toInteger())
}
udata.put("housing_age",db.housing_age)
udata.put("lift",db.lift.toString().toInteger())
udata.put("lift_household",db.lift_household.toString().toInteger())
@@ -83,7 +103,7 @@ class AddThirdPriceSample {
udata.put("room", db.room.toString().toInteger())
udata.put("room_structure",db.room_structure)
udata.put("source",db.source)
udata.put("total_price",db.total_price.toString().toInteger())
udata.put("total_price",db.total_price.toString().toDouble().toInteger())
udata.put("trade_date",db.trade_date)
udata.put("trade_property",db.trade_property)
udata.put("house_using",db.house_using)
@@ -98,11 +118,79 @@ class AddThirdPriceSample {
udata.put("match_id",db.match_id)
udata.put("price_flag", "国信达成交-推导")
DEAL_PRICE_COLL.save(udata)
// DEAL_PRICE_COLL.save(udata)
println index
// rets << udata
rets << udata
}
return rets
}
def matchOldData(){
def q = new BasicDBObject("source","贝壳找房").append("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2024-09-01","yyyy-MM-dd")))
def list = DEAL_PRICE_COLL.find(q).toList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(list.size())
for(def i=0;i<list.size();i++){
final def d = list[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def query = new BasicDBObject("url", d.url)
def obj = NO_PRICE_DEAL_PRICE_COLL.findOne(query)
if(obj){
def update = new BasicDBObject()
update.put("price_remark", "有价格")
NO_PRICE_DEAL_PRICE_COLL.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set', update), false,false)
}
println totalCount.get()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
def noInCcbCity(){
def citys = ["大理","苏州","无锡","通辽","绵阳","吉安","潍坊","襄阳","济宁","张家口","泸州","乐山","赣州","九江","眉山","桂林","雅安","广州","景德镇","上海","运城","内江","金华","濮阳","丹东","淄博","新乡","廊坊","烟台","邯郸","萍乡","宜宾","咸阳","许昌","德阳","银川","洛阳","天津","湛江","长沙","深圳","湘西","信阳","成都","威海","福州","上饶","常州","宜春","天水","平顶山","常德","吉林","盐城","杭州","赤峰","南京","周口","济南","株洲","大连","徐州","泰安","太原","凉山","兰州","承德","广元","台州","南昌","乌鲁木齐","海口","北京","淮安","清远","临沂","柳州","衢州","攀枝花","呼和浩特","昆明","岳阳","衡阳","温州","石家庄","连云港","保定","阜阳","遂宁","菏泽","青岛","唐山","开封","漳州","贵阳","驻马店","宝鸡","达州","汉中","南充","遵义","包头","南宁","沈阳","重庆","芜湖","厦门","大同"]
def q = new BasicDBObject()
q.put("price_remark",new BasicDBObject('$ne', "有价格"))
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2024-09-01","yyyy-MM-dd")))
q.put("avg_price",new BasicDBObject('$gt', 0))
q.put("match_id",new BasicDBObject('$exists', true))
def list = NO_PRICE_DEAL_PRICE_COLL.find(q).toList()
println "共:"+list.size()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(list.size())
for(def i=0;i<list.size();i++){
final def d = list[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
if(!citys.contains(d.city)){
def update = new BasicDBObject()
update.put("ccb_price_remark", "建行无此城市成交")
NO_PRICE_DEAL_PRICE_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set', update), false,false)
}
println totalCount.get()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
}
@@ -1234,15 +1234,7 @@ def findThirdCommCount(cityid, sqlInstance){
//}
//
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\无价格小区(3)2.23.xlsx"
//ExcelUtils.multiThreadingModify(fileName, { d ->
// def dist = CommonDatautils.findDistrictById(d['A'])
// if(dist){
// d['G'] = dist.city
// d['H'] = dist.region
// d['I'] = dist.name
// }
//})
@@ -1331,34 +1323,45 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
def fileName = "D:\\数据\\建行\\房产估值\\6个城市待优化_9月xlsx.xlsx"
//
ExcelUtils.multiThreadingModify(fileName, {d->
if(d['G']){
// def region = RegionData.formatRegion(d['A'], d['J'])
// if(region){
// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district")
//def fileName = "D:\\数据\\建行\\房产估值\\6个城市待优化_9月xlsx.xlsx"
////
//ExcelUtils.multiThreadingModify(fileName, {d->
// if(d['G']){
//// def region = RegionData.formatRegion(d['A'], d['J'])
//// if(region){
//// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district")
//// if(dist){
//// d['B'] = dist.region
//// d['C'] = dist.name
//// d['L'] = dist.status-dist.name_cat
//// }
//// }
// if(d['G'] instanceof String){
// def dist = CommonDatautils.findDistrictById(d['G'])
// if(dist){
// d['B'] = dist.region
// d['C'] = dist.name
// d['L'] = dist.status-dist.name_cat
//// d['A'] = dist.city
//// d['B'] = dist.region
//// d['C'] = dist.name
//// d['K'] = dist.estate_type2
//// d['L'] = dist.status
//// d['M'] = dist.name_cat
//
// d['H'] = dist.name
// }
// }
if(d['G'] instanceof String){
def dist = CommonDatautils.findDistrictById(d['G'])
if(dist){
// d['A'] = dist.city
// d['B'] = dist.region
// d['C'] = dist.name
// d['K'] = dist.estate_type2
// d['L'] = dist.status
// d['M'] = dist.name_cat
// }
//},0)
d['H'] = dist.name
}
}
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20241025人工指导价.xlsx"
ExcelUtils.multiThreadingModify(fileName, { d ->
def dist = CommonDatautils.findDistrictById(d['J'])
if(dist){
d['A'] = dist.city
d['B'] = dist.region
d['C'] = dist.name
}
},0)
})
//def file = "D:\\数据\\建行\\房产估值\\指导价\\5城价格翻新6-21.xlsx"
@@ -4785,7 +4785,6 @@ class CommonDatautils {
def q = new BasicDBObject()
q.put("type","区域")
q.put("district_diff",1)
q.put("city",new BasicDBObject('$in', ["深圳市"]))
def keys = new BasicDBObject()
keys.put("province",1)
@@ -4814,6 +4813,72 @@ class CommonDatautils {
return regionMap
}
static def queryAllRegionNameByCity(){
def q = new BasicDBObject()
q.put("type","区域")
q.put("district_diff",1)
def keys = new BasicDBObject()
keys.put("city_short",1)
keys.put("district_short",1)
keys.put("alias",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def retMap = [:]
for(def d : dbcursor){
def regionMap = retMap.get(d.city_short)
if(!regionMap){
regionMap = [:]
retMap.put(d.city_short, regionMap)
}
regionMap.put(d.district_short, d.alias)
}
return retMap
}
static def queryAllRegionFullNameByCity(){
def q = new BasicDBObject()
q.put("type","区域")
q.put("district_diff",1)
def keys = new BasicDBObject()
keys.put("city",1)
keys.put("district",1)
keys.put("alias",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def retMap = [:]
for(def d : dbcursor){
def regionMap = retMap.get(d.city)
if(!regionMap){
regionMap = [:]
retMap.put(d.city, regionMap)
}
regionMap.put(d.district, d.alias)
}
return retMap
}
static def queryAllStreetNameByCity(city){
def q = new BasicDBObject()
q.put("type","乡镇街道")
q.put("city", city)
def keys = new BasicDBObject()
keys.put("street",1)
keys.put("alias",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def retMap = [:]
for(def d : dbcursor){
retMap.put(d.street, d.alias)
}
return retMap
}
static def queryCommunityByCity(){
def q = new BasicDBObject()
@@ -49,7 +49,7 @@ class AutoAddDistrictBatch extends BaseBatch{
def fileName = "/home/goojia/gj/add_dist_info.xlsx"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\名录集\\第三方待入库数据.xlsx"
fileName = "D:\\数据\\数据建设\\小区\\第三方待入库数据.xlsx"
}
ExcelUtils.write(fileName, list, "待入库数据")
}
@@ -0,0 +1,210 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
/**
* 区域标准化
* 1.小区库区域标准化,包含小区,商铺,写字楼
* 2.poi数据库中 区域标准化
*/
class RegionStandardBatch extends BaseBatch{
def fullCityMap = CommonDatautils.queryFullNameCity()
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
static main(args) {
runBatch(new RegionStandardBatch(), args)
}
@Override
void process(Object args) {
// seaweedRegionStandard()
poiRegionStandard()
}
def seaweedRegionStandard(){
seaweedRegionStandardByCat("shop")
}
def poiRegionStandard(){
def citys = ["三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德"]//CommonDatautils.findAllCity()
for(def city : citys){
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
fullCityName = data.city
}
if(fullCityName){
def rMap = regionFullNameMap.get(fullCityName)
if(rMap){
def set = rMap.keySet()
def rets = findEsPoiByCity(fullCityName)
println city +" "+rets.size()
if(set.size()>1){
poiHaveRegionProcess(city, set, rMap, rets)
}else{
// haveStreetProcess(city, set, cat)
}
}
}else{
println "没找到相应的全称:"+city
}
}
}
def seaweedRegionStandardByCat(cat){
def citys = CommonDatautils.findAllCity()
for(def city : citys){
def rMap = regionMap.get(city)
if(rMap){
def set = rMap.keySet()
if(set.size()>1){
haveRegionProcess(city, set, rMap, cat)
}else{
haveStreetProcess(city, set, cat)
}
}
}
}
def haveRegionProcess(city, set, rMap, cat){
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name"])
for(def d : districts){
if(set.contains(d.region)){
continue
}
def updateRegion = null
for(def key : set){
def alias = rMap.get(key)
if(alias.contains(d.region)){
updateRegion = key
break;
}
}
if(updateRegion){
def udata = new BasicDBObject()
udata.put("region", updateRegion)
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name +" 区域修改为:"+updateRegion
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
}else{
def udata = new BasicDBObject()
udata.put("status", 1)
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name +" 状态修改为:删除"
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
}
}
}
def haveStreetProcess(city, set, cat){
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
fullCityName = data.city
}
def streetMap = CommonDatautils.queryAllStreetNameByCity(fullCityName)
if(streetMap){
set = streetMap.keySet()
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name"])
for(def d : districts){
if(set.contains(d.region)){
continue
}
def updateRegion = null
def flag = null
for(def key : set){
def alias = streetMap.get(key)
if((alias && alias.contains(d.region)) || (!alias && key.contains(d.region))){
def newRegion = RegionData.formatRegion(d.city, key)
if(newRegion!=d.region){
updateRegion = newRegion
}
flag = "pass"
break;
}
}
if(updateRegion){
def udata = new BasicDBObject()
udata.put("region", updateRegion)
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name +" 区域修改为:"+updateRegion
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
}else{
if(flag!="pass"){
def udata = new BasicDBObject()
udata.put("status", 1)
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name +" 状态修改为:删除"
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
}
}
}
}
}
def findEsPoiByCity(city){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
def rets = []
def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi_shards", queryBuilder)
for(def d: datas){
def o = new BasicDBObject()
o.put("city", d.city)
o.put("district", d.district)
o.put("name", d.name)
o.put("township", d.township)
o.put("location", d.location)
rets << o
})
return rets
}
def poiHaveRegionProcess(city, set, rMap, datas){
for(def d : datas){
if(set.contains(d.region)){
continue
}
def updateRegion = null
for(def key : set){
def alias = rMap.get(key)
if(alias.contains(d.region)){
updateRegion = key
break;
}
}
if(updateRegion){
def udata = new BasicDBObject()
udata.put("region", updateRegion)
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name +" 区域修改为:"+updateRegion
}else{
def udata = new BasicDBObject()
udata.put("status", 1)
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name +" 状态修改为:删除"
}
}
}
}
@@ -22,53 +22,85 @@ class TownCenterBatch extends BaseBatch{
@Override
void process(Object args) {
def shapes = findHasShapeTown()
def centers = findAllCenters()
checkCenter(shapes, centers)
def city = "上海市"
def shapes = findHasShapeTown(city)
def centers = findAllCenters(city)
def retMap = checkCenter(shapes, centers)
retMap.each {key,values->
if(values.size()>1){
println values
}
}
}
def checkCenter(shapes, centers){
def map = [:]
def index = 0
for(def town : shapes){
index ++
def points = InsidePolygon.createPoint4(town.shape.coordinates)
//1.先判断当前镇的中心点是否在围栏中
if(town.location){
def flag = InsidePolygon.isInArea(town.location.lng, town.location.lat, points)
def flag = InsidePolygon.isInArea(town.location.lon, town.location.lat, points)
if(!flag){
println town.city + " "+town.district+" "+town.street+" 不在本身围栏中"
}else{
def list = map.get(town.city + "#"+town.district+"#"+town.street)
if(!list){
list = []
map.put(town.city + "#"+town.district+"#"+town.street, list)
}
list << town.city + "#"+town.district+"#"+town.street
}
}
//2.判断是否有其它点在围栏中
for(def center : centers){
if(center._id.toString() != town._id.toString()){
def flag = InsidePolygon.isInArea(center.location.lng, center.location.lat, points)
def flag = InsidePolygon.isInArea(center.location.lon, center.location.lat, points)
if(flag){
def list = map.get(town.city + "#"+town.district+"#"+town.street)
if(!list){
list = []
map.put(town.city + "#"+town.district+"#"+town.street)
map.put(town.city + "#"+town.district+"#"+town.street, list)
}
list << center.city + "#"+center.district+"#"+center.street
println center.city + " "+center.district+" "+center.street+" 在 "+town.city + " "+town.district+" "+town.street+" 围栏中"
}
}
}
def list = map.get(town.city + "#"+town.district+"#"+town.street)
if(list){
println index+" "+town.city + "#"+town.district+"#"+town.street + " "+list.size()
}
}
return map
}
def findHasShapeTown(){
def findHasShapeTown(city){
def q = new BasicDBObject()
q.put("type", "乡镇街道")
q.put("shape", new BasicDBObject('$exists', true))
return ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(2000).toList()
if(city){
q.put("city", city)
}
def keys = new BasicDBObject()
keys.put("location", 1)
keys.put("city", 1)
keys.put("district", 1)
keys.put("street", 1)
keys.put("shape", 1)
return ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).toList()
}
def findAllCenters(){
def findAllCenters(city){
def q = new BasicDBObject()
q.put("type", "乡镇街道")
q.put("location", new BasicDBObject('$exists', true))
if(city){
q.put("city", city)
}
def keys = new BasicDBObject()
keys.put("location", 1)
keys.put("city", 1)
@@ -78,7 +78,7 @@ class TownMiddleProcess {
//对比中间表的街道与community街道是否一致,因为元极反馈类似“高密市醴泉街道”解析有问题,要去掉“高密市”,然后去了中间表
// updateCommunityDiffStreet()
// updateTownPoline()//更新乡镇街道围栏
updateTownPolineBatch()//批量更新乡镇街道围栏
// updateTownPolineBatch()//批量更新乡镇街道围栏
// checkTown()
//区域民政部名称更新到乡镇街道中
// updateTownshipByRegion()
@@ -94,6 +94,8 @@ class TownMiddleProcess {
// addTownCenter()//百度接口添加中心点
// checkCenterInTownRang()//检测乡镇街道中心点是否在围栏中
addSpecialCityTownAlias()
}
@@ -1301,6 +1303,57 @@ class TownMiddleProcess {
for(def d : dbcursor){
}
}
def addSpecialCityTownAlias(){
def fullCityMap = CommonDatautils.queryFullNameCity()
def citys = ["东莞","中山","嘉峪关","昆玉","石河子","阿拉尔","图木舒克","五家渠","济源","仙桃","潜江","天门","神农架","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中","北屯","铁门关","双河","可克达拉"]
def regionMap = CommonDatautils.getAllCityFullRegions()
for(def city : citys){
def townMap = regionMap.get(city)
def streets = findAllStreetByCity(city)
for(def s : streets){
def set = townMap.keySet()
def alias = null
for(def name : set){
def townObj = townMap.get(name)
if(townObj && townObj.pattern && townObj.pattern.contains(s.street)){
alias = townObj.pattern
break;
}
}
if(alias){
def newAlias = []
for(def n : alias){
if(n!=s.street){
newAlias << n
}
}
if(alias){
def udata = new BasicDBObject()
udata.put("alias", newAlias)
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id", s._id), new BasicDBObject('$set', udata),false, false)
}
}
}
}
}
def findAllStreetByCity(city){
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
fullCityName = data.city
}
def q = new BasicDBObject()
q.put("type", "乡镇街道")
q.put("city", fullCityName)
def keys = new BasicDBObject()
keys.put("street", 1)
return ADMINISTRATIVE_MAPPING_COLL.find(q, keys).toList()
}
}
@@ -42,7 +42,7 @@ class TownshipStrandUpdate {
// task.updateTown()
// task.stat()
// task.updateDiffRegionTown()
task.updateDiffRegionTown()
// task.cleanTown()
// task.checkTownFromMongo()
// task.updateTownMapping()
@@ -423,29 +423,29 @@ class TownshipStrandUpdate {
}
def updateDiffRegionTown(){
def fileName = "D:\\数据\\数据建设\\乡镇街道\\shop-不同区域.xlsx"
ExcelUtils.multiThreadingModify(fileName,{d->
def lists = CommonDatautils.findDistrictList(d['B'], null, d['D'], "shop")
def fileName = "D:\\数据\\数据建设\\乡镇街道\\district-不同区域.xlsx"
ExcelUtils.modify(fileName,{d->
def lists = CommonDatautils.findDistrictList(d['A'], null, d['C'], "district")
d['AA'] = lists.size()
//只处理库中只有一个的
if(lists && lists.size()==1 && d['Z']=="包含" && !d['AB']){
def newRegion = RegionData.formatRegion(d['B'], d['M'])
if(lists && lists.size()==1 && d['R']){
def newRegion = RegionData.formatRegion(d['A'], d['R'])
if(newRegion){
if(d['X'] && d['X'].toString().contains(d['M']+" "+d['N'])){ //地图街道在中间表中
// if(d['X'] && d['X'].toString().contains(d['M']+" "+d['N'])){ //地图街道在中间表中
def dbo = lists[0]
def arrs = d['X'].split(" ")
// def arrs = d['X'].split(" ")
def udata = new BasicDBObject()
udata.put("region", newRegion)
udata.put("m_date", new Date())
udata.put("township", arrs[1])
// udata.put("township", arrs[1])
DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata))
CommonDatautils.saveOptLog(dbo.city, dbo.region, dbo.name, "shop", "update","高娟")
CommonDatautils.saveOptLog(dbo.city, dbo.region, dbo.name, "district", "update-region","高娟")
d['AB'] = "已处理"
}
// }
}
}
})
},1)
}
def cleanTown(){
@@ -2,6 +2,7 @@ package com.fangjia.script.gj.poc
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
@@ -37,9 +38,9 @@ class AddPriceTest {
task.run();
}
def run(){
def filePath = "D:\\数据\\建行\\房产估值\\建行业务反馈badcase.xlsx"
// addGXDPrice(filePath)
addCcbPrice(filePath)
def filePath = "D:\\数据\\建行\\房产估值\\202410测试偏差较大案例统计数据test.xlsx"
addGXDPrice(filePath)
// addCcbPrice(filePath)
// addThirdPrice(filePath)
// addFjwPrice(filePath)
// exportPkPrice()
@@ -269,12 +270,16 @@ class AddPriceTest {
def addGXDPrice(filepath){
ExcelUtils.multiThreadingModify(filepath, { d ->
if (!d['M']) {
def dist = CommonDatautils.findDistrictById(d['B'])
if (!d['B']) {
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
// def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
d['M'] = gxdPrice
if(gxdPrice){
d['B'] = gxdPrice.price
d['C'] = DateUtils.formatDateByPattern(gxdPrice.c_date, "yyyy-MM-dd")
}
}
}
})
@@ -288,11 +293,11 @@ class AddPriceTest {
q.put("district_name", name)
q.put("source", "GXD")
def dbcursor = confirm_price_date.find(q).sort(new BasicDBObject("c_date", -1)).limit(1)
def price = null
def obj = null
for(def d : dbcursor){
price = d.price
obj = d
}
return price
return obj
}
@@ -236,6 +236,7 @@ class ElasticUtils89 {
searchRequest.scroll(scroll);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(qb);
searchSourceBuilder.size(100)
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
String scrollId = searchResponse.getScrollId();
@@ -247,14 +248,14 @@ class ElasticUtils89 {
searchHit.getSourceAsMap().put("id", searchHit.getId());
results << searchHit.getSourceAsMap()
}
if(results.size()%100000==0){
println "."+ new Date()
}
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
scrollId = searchResponse.getScrollId();
searchHits = searchResponse.getHits().getHits();
// if(results.size()%100000==0){
// println "."+ new Date()
// }
// SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
// scrollRequest.scroll(scroll);
// searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
// scrollId = searchResponse.getScrollId();
// searchHits = searchResponse.getHits().getHits();
}
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.addScrollId(scrollId);
@@ -150,8 +150,8 @@ class SeaweedDataSynch extends BaseBatch{
final CountDownLatch latch = new CountDownLatch(dists.size())
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
@@ -171,8 +171,8 @@ class SeaweedDataSynch extends BaseBatch{
} finally {
latch.countDown()
}
// }
// });
}
});
}
latch.await()
return rets;
@@ -414,7 +414,7 @@ public class ExcelUtils{
def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
def body = data.subList(1, data.size())
def nThread = Runtime.getRuntime().availableProcessors() * 4
def nThread = Runtime.getRuntime().availableProcessors() * 6
println nThread
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
final AtomicInteger counter = new AtomicInteger(0)