diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPriceSample.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPriceSample.groovy index a36d9d715..d3df8510d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPriceSample.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPriceSample.groovy @@ -1,11 +1,18 @@ package com.fangjia.script.ccbevaluation import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DateUtil +import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + /** * 将国信达及成交对标数据加入到第三方均价表,做为均价参考 */ @@ -17,18 +24,24 @@ class AddThirdPriceSample { public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + static main(args) { def task = new AddThirdPriceSample() - task.run() //添加第三方 fccs->保存的是国信达的参考价格 csjj->保存的是国信达的成交数据 -// task.addTrade() //添加成交 +// task.run() //添加第三方 fccs->保存的是国信达的参考价格 csjj->保存的是国信达的成交数据 + task.addTrade() //添加成交 + +// task.matchOldData() +// task.noInCcbCity() + } def run(){ - for(def i=0;i<19;i++){ - ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\国信达参考价作为第三方入库.xlsx",{d-> + for(def i=0;i<1;i++){ + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\国信达指导价录入-1024.xlsx",{d-> if(d['A'] && d['B']){ def dist = CommonDatautils.findDistrictById(d['A'].toString().trim()) if(dist){ - def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202408) + def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202409) // def udata = new BasicDBObject("csjj_esf_price", d['C'].toString().toDouble().toInteger()) def udata = new BasicDBObject("fccs_esf_price", d['B'].toString().toDouble().toInteger()) AVERAGE_STATISTICS.update(q,new BasicDBObject('$set', udata),false, false) @@ -44,15 +57,16 @@ class AddThirdPriceSample { } def addTrade(){ - def citys = ["武汉","南京","杭州","长沙","郑州","合肥","宁波","南通","嘉兴","珠海","泉州","绍兴","湖州","镇江"] - def rets = findTrades(citys) -// DEAL_PRICE_COLL.insert(rets) +// def citys = ["武汉","南京","杭州","长沙","郑州","合肥","宁波","南通","嘉兴","珠海","泉州","绍兴","湖州","镇江"] + def rets = findTrades() + DEAL_PRICE_COLL.insert(rets) } - def findTrades(citys){ + def findTrades(){ def q = new BasicDBObject() - q.put("city", new BasicDBObject('$in', citys)) - q.put("flag", 1) + q.put("ccb_price_remark","建行无此城市成交") +// q.put("city", new BasicDBObject('$in', citys)) +// q.put("flag", 1) def dbcursor = NO_PRICE_DEAL_PRICE_COLL.find(q) def rets = [] def index = 0 @@ -62,7 +76,7 @@ class AddThirdPriceSample { udata.put("_id",db._id) udata.put("url",db.url) - udata.put("area",db.area) + udata.put("area",db.area.toString().toDouble()) udata.put("avg_price",db.avg_price.toString().toDouble().toInteger()) udata.put("build_structure",db.build_structure) udata.put("build_type",db.build_type) @@ -73,8 +87,14 @@ class AddThirdPriceSample { udata.put("elevator",db.elevator) udata.put("fitment",db.fitment) udata.put("floor_info",db.floor_info) - udata.put("hall",db.hall.toString().toInteger()) - udata.put("height",db.height.toString().toInteger()) + if(db.hall){ + udata.put("hall",db.hall.toString().toInteger()) + } + + if(db.height){ + udata.put("height",db.height.toString().toInteger()) + } + udata.put("housing_age",db.housing_age) udata.put("lift",db.lift.toString().toInteger()) udata.put("lift_household",db.lift_household.toString().toInteger()) @@ -83,7 +103,7 @@ class AddThirdPriceSample { udata.put("room", db.room.toString().toInteger()) udata.put("room_structure",db.room_structure) udata.put("source",db.source) - udata.put("total_price",db.total_price.toString().toInteger()) + udata.put("total_price",db.total_price.toString().toDouble().toInteger()) udata.put("trade_date",db.trade_date) udata.put("trade_property",db.trade_property) udata.put("house_using",db.house_using) @@ -98,11 +118,79 @@ class AddThirdPriceSample { udata.put("match_id",db.match_id) udata.put("price_flag", "国信达成交-推导") - DEAL_PRICE_COLL.save(udata) +// DEAL_PRICE_COLL.save(udata) println index -// rets << udata + rets << udata } return rets } + def matchOldData(){ + def q = new BasicDBObject("source","贝壳找房").append("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2024-09-01","yyyy-MM-dd"))) + def list = DEAL_PRICE_COLL.find(q).toList() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(list.size()) + for(def i=0;i -// def dist = CommonDatautils.findDistrictById(d['A']) -// if(dist){ -// d['G'] = dist.city -// d['H'] = dist.region -// d['I'] = dist.name -// } -//}) + @@ -1331,34 +1323,45 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ -def fileName = "D:\\数据\\建行\\房产估值\\6个城市待优化_9月xlsx.xlsx" -// -ExcelUtils.multiThreadingModify(fileName, {d-> - if(d['G']){ -// def region = RegionData.formatRegion(d['A'], d['J']) -// if(region){ -// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district") +//def fileName = "D:\\数据\\建行\\房产估值\\6个城市待优化_9月xlsx.xlsx" +//// +//ExcelUtils.multiThreadingModify(fileName, {d-> +// if(d['G']){ +//// def region = RegionData.formatRegion(d['A'], d['J']) +//// if(region){ +//// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district") +//// if(dist){ +//// d['B'] = dist.region +//// d['C'] = dist.name +//// d['L'] = dist.status-dist.name_cat +//// } +//// } +// if(d['G'] instanceof String){ +// def dist = CommonDatautils.findDistrictById(d['G']) // if(dist){ -// d['B'] = dist.region -// d['C'] = dist.name -// d['L'] = dist.status-dist.name_cat +//// d['A'] = dist.city +//// d['B'] = dist.region +//// d['C'] = dist.name +//// d['K'] = dist.estate_type2 +//// d['L'] = dist.status +//// d['M'] = dist.name_cat +// +// d['H'] = dist.name // } // } - if(d['G'] instanceof String){ - def dist = CommonDatautils.findDistrictById(d['G']) - if(dist){ -// d['A'] = dist.city -// d['B'] = dist.region -// d['C'] = dist.name -// d['K'] = dist.estate_type2 -// d['L'] = dist.status -// d['M'] = dist.name_cat +// } +//},0) - d['H'] = dist.name - } - } + +def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20241025人工指导价.xlsx" +ExcelUtils.multiThreadingModify(fileName, { d -> + def dist = CommonDatautils.findDistrictById(d['J']) + if(dist){ + d['A'] = dist.city + d['B'] = dist.region + d['C'] = dist.name } -},0) +}) //def file = "D:\\数据\\建行\\房产估值\\指导价\\5城价格翻新6-21.xlsx" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 5e2937768..cc8d27021 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -4785,7 +4785,6 @@ class CommonDatautils { def q = new BasicDBObject() q.put("type","区域") q.put("district_diff",1) - q.put("city",new BasicDBObject('$in', ["深圳市"])) def keys = new BasicDBObject() keys.put("province",1) @@ -4814,6 +4813,72 @@ class CommonDatautils { return regionMap } + + static def queryAllRegionNameByCity(){ + + def q = new BasicDBObject() + q.put("type","区域") + q.put("district_diff",1) + + def keys = new BasicDBObject() + keys.put("city_short",1) + keys.put("district_short",1) + keys.put("alias",1) + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def retMap = [:] + for(def d : dbcursor){ + def regionMap = retMap.get(d.city_short) + if(!regionMap){ + regionMap = [:] + retMap.put(d.city_short, regionMap) + } + regionMap.put(d.district_short, d.alias) + } + + return retMap + } + + static def queryAllRegionFullNameByCity(){ + def q = new BasicDBObject() + q.put("type","区域") + q.put("district_diff",1) + + def keys = new BasicDBObject() + keys.put("city",1) + keys.put("district",1) + keys.put("alias",1) + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def retMap = [:] + for(def d : dbcursor){ + def regionMap = retMap.get(d.city) + if(!regionMap){ + regionMap = [:] + retMap.put(d.city, regionMap) + } + regionMap.put(d.district, d.alias) + } + + return retMap + } + + static def queryAllStreetNameByCity(city){ + + def q = new BasicDBObject() + q.put("type","乡镇街道") + q.put("city", city) + + def keys = new BasicDBObject() + keys.put("street",1) + keys.put("alias",1) + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def retMap = [:] + for(def d : dbcursor){ + retMap.put(d.street, d.alias) + } + + return retMap + } + static def queryCommunityByCity(){ def q = new BasicDBObject() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy index f27ee361f..f6a8c1208 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy @@ -49,7 +49,7 @@ class AutoAddDistrictBatch extends BaseBatch{ def fileName = "/home/goojia/gj/add_dist_info.xlsx" if (CommonDatautils.isWindow()) { - fileName = "D:\\数据\\数据建设\\名录集\\第三方待入库数据.xlsx" + fileName = "D:\\数据\\数据建设\\小区\\第三方待入库数据.xlsx" } ExcelUtils.write(fileName, list, "待入库数据") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy new file mode 100644 index 000000000..3bc2f080a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy @@ -0,0 +1,210 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.core.RegionData +import com.fangjia.script.BaseBatch +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.xiamen.ElasticUtils89 +import com.fangjia.script.util.DbUtils +import com.mongodb.BasicDBObject +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +/** + * 区域标准化 + * 1.小区库区域标准化,包含小区,商铺,写字楼 + * 2.poi数据库中 区域标准化 + */ +class RegionStandardBatch extends BaseBatch{ + + def fullCityMap = CommonDatautils.queryFullNameCity() + def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity() + def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity() + + static main(args) { + runBatch(new RegionStandardBatch(), args) + } + + @Override + void process(Object args) { +// seaweedRegionStandard() + poiRegionStandard() + } + + def seaweedRegionStandard(){ + seaweedRegionStandardByCat("shop") + } + def poiRegionStandard(){ + def citys = ["三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德"]//CommonDatautils.findAllCity() + for(def city : citys){ + + def data = fullCityMap.get(city) + def fullCityName = null + if(data){ + fullCityName = data.city + } + if(fullCityName){ + + def rMap = regionFullNameMap.get(fullCityName) + if(rMap){ + def set = rMap.keySet() + def rets = findEsPoiByCity(fullCityName) + println city +" "+rets.size() + if(set.size()>1){ + poiHaveRegionProcess(city, set, rMap, rets) + }else{ +// haveStreetProcess(city, set, cat) + } + } + }else{ + println "没找到相应的全称:"+city + } + + } + + } + + + def seaweedRegionStandardByCat(cat){ + def citys = CommonDatautils.findAllCity() + for(def city : citys){ + + def rMap = regionMap.get(city) + if(rMap){ + def set = rMap.keySet() + if(set.size()>1){ + haveRegionProcess(city, set, rMap, cat) + }else{ + haveStreetProcess(city, set, cat) + } + } + } + + } + + def haveRegionProcess(city, set, rMap, cat){ + def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name"]) + for(def d : districts){ + if(set.contains(d.region)){ + continue + } + def updateRegion = null + for(def key : set){ + def alias = rMap.get(key) + if(alias.contains(d.region)){ + updateRegion = key + break; + } + } + if(updateRegion){ + def udata = new BasicDBObject() + udata.put("region", updateRegion) + udata.put("m_date", new Date()) + println d.city +" "+d.region+" "+d.name +" 区域修改为:"+updateRegion + DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + }else{ + def udata = new BasicDBObject() + udata.put("status", 1) + udata.put("m_date", new Date()) + println d.city +" "+d.region+" "+d.name +" 状态修改为:删除" + DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + } + } + } + + def haveStreetProcess(city, set, cat){ + def data = fullCityMap.get(city) + def fullCityName = null + if(data){ + fullCityName = data.city + } + def streetMap = CommonDatautils.queryAllStreetNameByCity(fullCityName) + if(streetMap){ + set = streetMap.keySet() + def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name"]) + for(def d : districts){ + if(set.contains(d.region)){ + continue + } + + def updateRegion = null + def flag = null + for(def key : set){ + def alias = streetMap.get(key) + if((alias && alias.contains(d.region)) || (!alias && key.contains(d.region))){ + def newRegion = RegionData.formatRegion(d.city, key) + if(newRegion!=d.region){ + updateRegion = newRegion + } + flag = "pass" + break; + } + } + if(updateRegion){ + def udata = new BasicDBObject() + udata.put("region", updateRegion) + udata.put("m_date", new Date()) + println d.city +" "+d.region+" "+d.name +" 区域修改为:"+updateRegion +// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + }else{ + if(flag!="pass"){ + def udata = new BasicDBObject() + udata.put("status", 1) + udata.put("m_date", new Date()) + println d.city +" "+d.region+" "+d.name +" 状态修改为:删除" +// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + } + } + } + } + } + + def findEsPoiByCity(city){ + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) + def rets = [] + + def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi_shards", queryBuilder) + for(def d: datas){ + def o = new BasicDBObject() + o.put("city", d.city) + o.put("district", d.district) + o.put("name", d.name) + o.put("township", d.township) + o.put("location", d.location) + rets << o + }) + return rets + } + + + def poiHaveRegionProcess(city, set, rMap, datas){ + for(def d : datas){ + if(set.contains(d.region)){ + continue + } + def updateRegion = null + for(def key : set){ + def alias = rMap.get(key) + if(alias.contains(d.region)){ + updateRegion = key + break; + } + } + if(updateRegion){ + def udata = new BasicDBObject() + udata.put("region", updateRegion) + udata.put("m_date", new Date()) + println d.city +" "+d.region+" "+d.name +" 区域修改为:"+updateRegion + + }else{ + def udata = new BasicDBObject() + udata.put("status", 1) + udata.put("m_date", new Date()) + println d.city +" "+d.region+" "+d.name +" 状态修改为:删除" + + } + } + } + + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownCenterBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownCenterBatch.groovy index e1bfa8969..b776efc53 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownCenterBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownCenterBatch.groovy @@ -22,53 +22,85 @@ class TownCenterBatch extends BaseBatch{ @Override void process(Object args) { - def shapes = findHasShapeTown() - def centers = findAllCenters() - checkCenter(shapes, centers) + def city = "上海市" + def shapes = findHasShapeTown(city) + def centers = findAllCenters(city) + def retMap = checkCenter(shapes, centers) + retMap.each {key,values-> + if(values.size()>1){ + println values + } + } } def checkCenter(shapes, centers){ def map = [:] + def index = 0 for(def town : shapes){ + index ++ def points = InsidePolygon.createPoint4(town.shape.coordinates) //1.先判断当前镇的中心点是否在围栏中 if(town.location){ - def flag = InsidePolygon.isInArea(town.location.lng, town.location.lat, points) + def flag = InsidePolygon.isInArea(town.location.lon, town.location.lat, points) if(!flag){ println town.city + " "+town.district+" "+town.street+" 不在本身围栏中" + }else{ + def list = map.get(town.city + "#"+town.district+"#"+town.street) + if(!list){ + list = [] + map.put(town.city + "#"+town.district+"#"+town.street, list) + } + list << town.city + "#"+town.district+"#"+town.street } } //2.判断是否有其它点在围栏中 for(def center : centers){ if(center._id.toString() != town._id.toString()){ - def flag = InsidePolygon.isInArea(center.location.lng, center.location.lat, points) + def flag = InsidePolygon.isInArea(center.location.lon, center.location.lat, points) if(flag){ def list = map.get(town.city + "#"+town.district+"#"+town.street) if(!list){ list = [] - map.put(town.city + "#"+town.district+"#"+town.street) + map.put(town.city + "#"+town.district+"#"+town.street, list) } list << center.city + "#"+center.district+"#"+center.street println center.city + " "+center.district+" "+center.street+" 在 "+town.city + " "+town.district+" "+town.street+" 围栏中" } } } + def list = map.get(town.city + "#"+town.district+"#"+town.street) + if(list){ + println index+" "+town.city + "#"+town.district+"#"+town.street + " "+list.size() + } } return map } - def findHasShapeTown(){ + def findHasShapeTown(city){ def q = new BasicDBObject() q.put("type", "乡镇街道") q.put("shape", new BasicDBObject('$exists', true)) - return ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(2000).toList() + if(city){ + q.put("city", city) + } + + def keys = new BasicDBObject() + keys.put("location", 1) + keys.put("city", 1) + keys.put("district", 1) + keys.put("street", 1) + keys.put("shape", 1) + return ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).toList() } - def findAllCenters(){ + def findAllCenters(city){ def q = new BasicDBObject() q.put("type", "乡镇街道") q.put("location", new BasicDBObject('$exists', true)) + if(city){ + q.put("city", city) + } def keys = new BasicDBObject() keys.put("location", 1) keys.put("city", 1) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy index 9d5283a6b..4ecd49ed0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy @@ -78,7 +78,7 @@ class TownMiddleProcess { //对比中间表的街道与community街道是否一致,因为元极反馈类似“高密市醴泉街道”解析有问题,要去掉“高密市”,然后去了中间表 // updateCommunityDiffStreet() // updateTownPoline()//更新乡镇街道围栏 - updateTownPolineBatch()//批量更新乡镇街道围栏 +// updateTownPolineBatch()//批量更新乡镇街道围栏 // checkTown() //区域民政部名称更新到乡镇街道中 // updateTownshipByRegion() @@ -94,6 +94,8 @@ class TownMiddleProcess { // addTownCenter()//百度接口添加中心点 // checkCenterInTownRang()//检测乡镇街道中心点是否在围栏中 + + addSpecialCityTownAlias() } @@ -1301,6 +1303,57 @@ class TownMiddleProcess { for(def d : dbcursor){ } + } + def addSpecialCityTownAlias(){ + def fullCityMap = CommonDatautils.queryFullNameCity() + + def citys = ["东莞","中山","嘉峪关","昆玉","石河子","阿拉尔","图木舒克","五家渠","济源","仙桃","潜江","天门","神农架","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中","北屯","铁门关","双河","可克达拉"] + def regionMap = CommonDatautils.getAllCityFullRegions() + for(def city : citys){ + def townMap = regionMap.get(city) + def streets = findAllStreetByCity(city) + for(def s : streets){ + def set = townMap.keySet() + def alias = null + for(def name : set){ + def townObj = townMap.get(name) + if(townObj && townObj.pattern && townObj.pattern.contains(s.street)){ + alias = townObj.pattern + break; + } + } + if(alias){ + def newAlias = [] + for(def n : alias){ + if(n!=s.street){ + newAlias << n + } + } + if(alias){ + def udata = new BasicDBObject() + udata.put("alias", newAlias) + ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id", s._id), new BasicDBObject('$set', udata),false, false) + } + } + } + } + } + + + def findAllStreetByCity(city){ + def data = fullCityMap.get(city) + def fullCityName = null + if(data){ + fullCityName = data.city + } + + def q = new BasicDBObject() + q.put("type", "乡镇街道") + q.put("city", fullCityName) + + def keys = new BasicDBObject() + keys.put("street", 1) + return ADMINISTRATIVE_MAPPING_COLL.find(q, keys).toList() } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy index 2a6c1d7a3..d99e249c9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy @@ -42,7 +42,7 @@ class TownshipStrandUpdate { // task.updateTown() // task.stat() -// task.updateDiffRegionTown() + task.updateDiffRegionTown() // task.cleanTown() // task.checkTownFromMongo() // task.updateTownMapping() @@ -423,29 +423,29 @@ class TownshipStrandUpdate { } def updateDiffRegionTown(){ - def fileName = "D:\\数据\\数据建设\\乡镇街道\\shop-不同区域.xlsx" - ExcelUtils.multiThreadingModify(fileName,{d-> - def lists = CommonDatautils.findDistrictList(d['B'], null, d['D'], "shop") + def fileName = "D:\\数据\\数据建设\\乡镇街道\\district-不同区域.xlsx" + ExcelUtils.modify(fileName,{d-> + def lists = CommonDatautils.findDistrictList(d['A'], null, d['C'], "district") d['AA'] = lists.size() //只处理库中只有一个的 - if(lists && lists.size()==1 && d['Z']=="包含" && !d['AB']){ - def newRegion = RegionData.formatRegion(d['B'], d['M']) + if(lists && lists.size()==1 && d['R']){ + def newRegion = RegionData.formatRegion(d['A'], d['R']) if(newRegion){ - if(d['X'] && d['X'].toString().contains(d['M']+" "+d['N'])){ //地图街道在中间表中 +// if(d['X'] && d['X'].toString().contains(d['M']+" "+d['N'])){ //地图街道在中间表中 def dbo = lists[0] - def arrs = d['X'].split(" ") +// def arrs = d['X'].split(" ") def udata = new BasicDBObject() udata.put("region", newRegion) udata.put("m_date", new Date()) - udata.put("township", arrs[1]) +// udata.put("township", arrs[1]) DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata)) - CommonDatautils.saveOptLog(dbo.city, dbo.region, dbo.name, "shop", "update","高娟") + CommonDatautils.saveOptLog(dbo.city, dbo.region, dbo.name, "district", "update-region","高娟") d['AB'] = "已处理" - } +// } } } - }) + },1) } def cleanTown(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy index eed8d9fda..d314427a9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy @@ -2,6 +2,7 @@ package com.fangjia.script.gj.poc import com.fangjia.script.ccbevaluation.CcbCommonUtils import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DateUtil import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils @@ -37,9 +38,9 @@ class AddPriceTest { task.run(); } def run(){ - def filePath = "D:\\数据\\建行\\房产估值\\建行业务反馈badcase.xlsx" -// addGXDPrice(filePath) - addCcbPrice(filePath) + def filePath = "D:\\数据\\建行\\房产估值\\202410测试偏差较大案例统计数据test.xlsx" + addGXDPrice(filePath) +// addCcbPrice(filePath) // addThirdPrice(filePath) // addFjwPrice(filePath) // exportPkPrice() @@ -269,12 +270,16 @@ class AddPriceTest { def addGXDPrice(filepath){ ExcelUtils.multiThreadingModify(filepath, { d -> - if (!d['M']) { - def dist = CommonDatautils.findDistrictById(d['B']) + if (!d['B']) { + def dist = CommonDatautils.findDistrictById(d['A']) if(dist){ def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name) // def gxdPrice = findGXDPrice(d['D'], d['E'], d['C']) - d['M'] = gxdPrice + if(gxdPrice){ + d['B'] = gxdPrice.price + d['C'] = DateUtils.formatDateByPattern(gxdPrice.c_date, "yyyy-MM-dd") + } + } } }) @@ -288,11 +293,11 @@ class AddPriceTest { q.put("district_name", name) q.put("source", "GXD") def dbcursor = confirm_price_date.find(q).sort(new BasicDBObject("c_date", -1)).limit(1) - def price = null + def obj = null for(def d : dbcursor){ - price = d.price + obj = d } - return price + return obj } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy index 2d2f76f24..db5362ca7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy @@ -236,6 +236,7 @@ class ElasticUtils89 { searchRequest.scroll(scroll); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query(qb); + searchSourceBuilder.size(100) searchRequest.source(searchSourceBuilder); SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); String scrollId = searchResponse.getScrollId(); @@ -247,14 +248,14 @@ class ElasticUtils89 { searchHit.getSourceAsMap().put("id", searchHit.getId()); results << searchHit.getSourceAsMap() } - if(results.size()%100000==0){ - println "."+ new Date() - } - SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); - scrollRequest.scroll(scroll); - searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT); - scrollId = searchResponse.getScrollId(); - searchHits = searchResponse.getHits().getHits(); +// if(results.size()%100000==0){ +// println "."+ new Date() +// } +// SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); +// scrollRequest.scroll(scroll); +// searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT); +// scrollId = searchResponse.getScrollId(); +// searchHits = searchResponse.getHits().getHits(); } ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); clearScrollRequest.addScrollId(scrollId); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy index 5b5e79b62..e78bc7018 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy @@ -150,8 +150,8 @@ class SeaweedDataSynch extends BaseBatch{ final CountDownLatch latch = new CountDownLatch(dists.size()) for(def i=0;i