From 5ef49f615da622a5510d1eb49bbd29c7a07ed6f2 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 15 May 2024 16:41:40 +0800 Subject: [PATCH] update --- .../ccbevaluation/AddThirdPartyPrice.groovy | 2 +- .../ccbevaluation/BigdataBuildingBatch.groovy | 2 +- .../BigdataCommunityBatch.groovy | 2 +- .../ccbevaluation/BigdataDealBatch.groovy | 2 +- .../ccbevaluation/BigdataForsaleBatch.groovy | 2 +- .../ccbevaluation/CcbCommonUtils.groovy | 15 +- .../DistrictGuidePriceBatch.groovy | 2 +- .../ccbevaluation/GuiDingPriceBatch.groovy | 4 +- .../ccbevaluation/StagingDistrictBatch.groovy | 2 +- .../fangjia/script/ccbevaluation/Test.groovy | 17 +- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 2 +- .../ronghe/BigdataCitypriceBatch.groovy | 2 +- .../script/commData/CommonDatautils.groovy | 29 +- .../script/gj/data_handle/DevProcess.groovy | 353 +++++++++++++++++- .../data_handle/DistrictAliasProcess.groovy | 45 +++ .../gj/data_handle/TownMiddleProcess.groovy | 44 ++- .../gj/district/FangjiaClientUtils.groovy | 2 +- .../FjwBigdataCommbuildtypeBatch.groovy | 137 +++++++ .../FjwBigdataCommunityBatch.groovy | 324 ++++++++++++++++ .../script/gj/meishubao/MsbPocTest.groovy | 44 ++- .../fangjia/script/gj/poc/AddPriceTest.groovy | 16 +- .../script/gj/village/QiliangTest.groovy | 4 +- .../com/fangjia/script/util/ExcelUtils.groovy | 2 +- 23 files changed, 988 insertions(+), 66 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictAliasProcess.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommbuildtypeBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommunityBatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPartyPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPartyPrice.groovy index ca14ea514..e0da72624 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPartyPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPartyPrice.groovy @@ -23,7 +23,7 @@ class AddThirdPartyPrice { def cityCodeDatas = CcbCommonUtils.findCityCodeValue() for(def c : citys){ println c +" "+cityCodeDatas.get(c) - def dists = CcbCommonUtils.findAllDistrictByCity(c, "district", ["city", "region", "name","name_cat", "estate_type2","lng"]) + def dists = CcbCommonUtils.findAllDistrictByCity(c, "district", ["city", "region", "name","name_cat", "estate_type2","lng"], null) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(c) def rets = createDate(dists, distCodeMap) println dists.size() +" "+rets.size() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy index 431972b91..2eda22bba 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy @@ -47,7 +47,7 @@ class BigdataBuildingBatch extends BaseBatch{ def getDistrictMap(def city){ - def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias","build_type","build_structure"]); + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias","build_type","build_structure"], null); def distMap = [:] for(def d : districts){ def alias = d.alias; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 0ea40b8c2..47159ee0e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -58,7 +58,7 @@ class BigdataCommunityBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"]) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], null) def ccbMaps = null diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 05a5447e8..a45e49793 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -529,7 +529,7 @@ class BigdataDealBatch extends BaseBatch{ def getCityData(def city, def statMap, def ppeDistsMap){ - def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"]) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"], null) println city +" 小区量:"+dists.size() def tradesList = Collections.synchronizedList(new ArrayList()); def totalCount = new AtomicLong() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 8b444eaba..e329c8732 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -97,7 +97,7 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], null); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219); exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, startDate.toInteger(), endDate.toInteger(), houseIdsfilter); //测试通过匹配接口配房源效果 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 65f84ab77..a1f03acd8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -428,7 +428,7 @@ class CcbCommonUtils { return rows } - public static def findAllDistrictByCity(def city, def cat, def fields){ + public static def findAllDistrictByCity(def city, def cat, def fields, def kind){ DBObject q = new BasicDBObject(); q.put("status",0) @@ -445,14 +445,13 @@ class CcbCommonUtils { keys.put(field, 1); } } -// def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000) def dbsour = SEAWEED_MONGO_COLL.find(q,keys).batchSize(2000) println "[${city}] 原始:"+dbsour.count() def rets = [] def ids = [] for(def d : dbsour){ - if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2) && !d.name.contains("洋房") && !d.name.contains("公房")){ + if(kind){ if(d.address){ d.address = d.address.replaceAll("\r|\n|\\s", ""); } @@ -460,6 +459,16 @@ class CcbCommonUtils { ids << d._id.toString() rets << d } + }else{ + if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2) && !d.name.contains("洋房") && !d.name.contains("公房")){ + if(d.address){ + d.address = d.address.replaceAll("\r|\n|\\s", ""); + } + if(!ids.contains(d._id.toString())){ + ids << d._id.toString() + rets << d + } + } } } println "[${city}] 小区量:"+rets.size() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy index ed5ed5f03..fdf28955b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy @@ -41,7 +41,7 @@ class DistrictGuidePriceBatch extends BaseBatch{ def getDistrictMap(def city){ - def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]); + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"], null); def distMap = [:] for(def d : districts){ distMap.put(d.region+"##"+d.name, d) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 99bed78ed..9ad3f9a31 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2024-03-27", "2024-04-26") + def prices = CcbCommonUtils.findPersonPrice(city, "2024-04-26", "2024-05-10") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) @@ -207,7 +207,7 @@ class GuiDingPriceBatch extends BaseBatch{ } def getDistrictMap(def city){ - def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]); + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"], null); def distMap = [:] for(def d : districts){ distMap.put(d.region+"##"+d.name, d) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy index cae04cf55..82d2c96c0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy @@ -31,7 +31,7 @@ class StagingDistrictBatch extends BaseBatch{ def citys = CcbCommonUtils.getRunningCitys(args) def rows = [] for(def city : citys){ - def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","lng","master_district_name", "c_date", "m_date"]) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","lng","master_district_name", "c_date", "m_date"], null) def fenQiMap = getFenQiMap(dists); def rets = createCityData(dists, fenQiMap) rows.addAll(rets) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 371e260ea..5234f7fd4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -1289,10 +1289,10 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ -def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240425--人工指导价.xls" +def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240508--人工指导价.xls" ExcelUtils.multiThreadingModify(fileName, {d-> - if(d['C']){ + if(d['J']){ // def region = RegionData.formatRegion(d['B'], d['C']) // if(region){ // def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district") @@ -1301,12 +1301,15 @@ ExcelUtils.multiThreadingModify(fileName, {d-> // d['H'] = dist._id.toString() // } // } - if(!d['L']){ - def dist = CommonDatautils.findDistrictById(d['C']) + if(!d['A']){ + def dist = CommonDatautils.findDistrictById(d['J']) if(dist){ - d['L'] = dist.region - d['M'] = dist.name - d['N'] = dist.status + d['A'] = dist.city + d['B'] = dist.region + d['C'] = dist.name + d['K'] = dist.estate_type2 + d['L'] = dist.status + d['M'] = dist.name_cat } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 8483a84a0..b0756a439 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -71,7 +71,7 @@ class ThirdPartyPriceBatch extends BaseBatch { def statMap = getDistrictStatByCity(city, weekend) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"]) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"], null) def thirdPriceMap = findThirdPartyPrice(city, statMap) //def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap)//老第三方均价表(现不用) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index 5a38c237e..479813af4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -207,7 +207,7 @@ class BigdataCitypriceBatch extends BaseBatch{ def allLastMonthMap = [:] def allLastYearMap = [:] for(def city : citys) { - def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias"]); + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias"], null); allDistricts.addAll(districts) def priceMap = CommonDatautils.queryDistrictStatByDate(city, startDate, endDate) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 233a8e0f7..fde3609ec 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1720,7 +1720,34 @@ class CommonDatautils { } return regionMap; } - + + + public static def getAllCityFullRegions(){ + DBObject queryCondition = new BasicDBObject() + queryCondition.put("category","region"); + //queryCondition.put("show",1); + def dbCursor = REGION_BLOCK_COLL.find(queryCondition) + def regionMap = [:]; + for(DBObject obj : dbCursor){ + def city = obj.get("city"); + def region = obj.get("name"); + def pattern = obj.get("pattern"); + if(region.indexOf("周边")==-1 && region.indexOf("其他")==-1) { + def r_map = regionMap.get(city); + if(!r_map){ + r_map = [:] + regionMap.put(city, r_map); + } + + if(!pattern){ + pattern = []; + pattern << region; + } + r_map.put(region, ["pattern":pattern, "full_region":obj.get("org_name")]); + } + } + return regionMap; + } public static def getRegionsByCity(def city){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index 90d781a2f..342e946fb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -1,25 +1,44 @@ package com.fangjia.script.gj.data_handle import com.fangjia.core.RegionData +import com.fangjia.core.util.GeoUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.linxsh.util.AddressUtil import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBObject +import com.mongodb.Bytes import com.mongodb.DBCollection import net.sf.json.JSONObject import org.apache.poi.ss.util.RegionUtil +import org.bson.types.ObjectId class DevProcess { def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL) + DBCollection DEV_MAPPING_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL) + static main(args) { def task = new DevProcess(); // task.run() - task.parseCityAndDist() +// task.parseCityAndDist() +// task.devSplitRoad() +// task.addFlag() +// task.matchDev() + +// task.calAddrDistince() +// task.updateCityAndDist() +// task.addDevAddres1() //按添加地址中包含开发区名称的地址 +// task.addDevAddres2()//添加相差2公里范围内的地址 + +// task.stat() +// task.addNewDev() +// task.createDevMapping() + task.deleteAddr() } def parseCityAndDist(){ @@ -131,10 +150,336 @@ class DevProcess { }else{ println udata.name +" 已存在" } - - } - } } + + def devSplitRoad(){ + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\企业园区地址-新.xlsx",{d-> + /*if(d['C'] && !d['V']){ + def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['E'], d['C']) + def street = jsonObject.get("street") + def streetNumber = jsonObject.get("streetNumber") + if(jsonObject.get("province") && jsonObject.get("province")!="null"){ + d['V'] = jsonObject.get("province") + } + + if(jsonObject.get("city") && jsonObject.get("city")!="null"){ + d['W'] = jsonObject.get("city") + } + + if(jsonObject.get("district") && jsonObject.get("district")!="null"){ + d['X'] = jsonObject.get("district") + } + + if(jsonObject && street && streetNumber && street!="null"){ + d['Y'] = jsonObject.get("street") + if(streetNumber!='null'){ + d['Z'] = jsonObject.get("streetNumber") + } + } + }*/ + if(d['D'] && !d['M'] && d['P'] && d['P'].toString().toDouble().toInteger()==1){ + def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['E'], d['D']) + def street = jsonObject.get("street") + def streetNumber = jsonObject.get("streetNumber") + if(jsonObject.get("province") && jsonObject.get("province")!="null"){ + d['J'] = jsonObject.get("province") + } + + if(jsonObject.get("city") && jsonObject.get("city")!="null"){ + d['K'] = jsonObject.get("city") + } + + if(jsonObject.get("district") && jsonObject.get("district")!="null"){ + d['L'] = jsonObject.get("district") + } + + if(jsonObject && street && streetNumber && street!="null"){ + d['M'] = jsonObject.get("street") + if(streetNumber!='null'){ + d['N'] = jsonObject.get("streetNumber") + } + } + } + },0) + } + + def addFlag(){ + ExcelUtils.modify("C:\\Users\\gaojuan\\Downloads\\企业园区地址.xlsx",{d-> + if(d['H'] && d['D']){ + if(d['D'].contains(d['H'])){ + d['O'] = "是" + }else{ + def newName = null + if(d['H'].toString().startsWith(d['I'])){ + newName = d['H'].toString().substring(d['I'].toString().length()) + }else if(d['H'].toString().startsWith(d['I']+"省")){ + newName = d['H'].toString().substring(d['I'].toString().length()+1) + } + if(newName && d['D'].contains(newName)){ + d['O'] = "是2" + } + } + } + },1); + } + + def matchDev(){ + ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Downloads\\企业园区地址.xlsx",{d-> + if(d['H']){ + def q = new BasicDBObject() + q.put("name", d['H']) + def obj = development_zone_CONN.findOne(q) + if(obj){ + d['S'] = obj._id.toString() + d['T'] = obj.name + } + } + },1); + } + + def calAddrDistince(){ + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\企业园区地址-新.xlsx",{d-> + if(d['M'] && d['N'] && d['Y'] && d['Z']){ + def addr1 = getFullAddress(d['J'], d['K'],d['L'],d['M'] , d['N']) + def addr2 = getFullAddress(d['V'], d['W'], d['X'], d['Y'] , d['Z']) + + def json1 = GaoDeApiUtils.geo(d['K'], addr1) + def json2 = GaoDeApiUtils.geo(d['W'], addr2) + def arrs1 = getLocation(json1) + def arrs2 = getLocation(json2) + if(arrs1 && arrs2){ + def distance = GeoUtils.calculateDistance(arrs1[0].toString().toDouble(), arrs1[1].toString().toDouble(), arrs2[0].toString().toDouble() , arrs2[1].toString().toDouble() ) + d['AA'] = distance + } + } + },0); + } + + def getFullAddress(province, city, region, street, streetName){ + def addr = "" + if(addr){ + addr = province + } + if(city){ + addr = addr + city + } + if(region){ + addr = addr + region + } + addr = addr + street + streetName + return addr + } + def getLocation(json1){ + def arrs1 = null + if(json1 && json1.status=="1") { + def ret = json1.geocodes + if (ret) { + def str = ret.get(0).location + if(str){ + arrs1 = str.split(",") + } + } + } + return arrs1 + } + + def updateCityAndDist(){ + ExcelUtils.modify("D:\\数据\\数据建设\\等级开发区 - 汇总.xlsx",{d-> + if(d['L']){ + def udata = new BasicDBObject() + if(d['L']=="采用拆分结果"){ + if(d["I"]){ + udata.put("province", d["I"]) + } + if(d["J"]){ + udata.put("city", d["J"]) + } + if(d["K"]){ + udata.put("district", d["K"]) + } + }else if(d['L']=="采用地图结果"){ + if(d["H"]){ + udata.put("province", d["H"]) + } + if(d["E"]){ + udata.put("city", d["E"]) + } + if(d["F"]){ + udata.put("district", d["F"]) + } + } + + def obj = development_zone_CONN.findOne(new BasicDBObject("name", d['C'])) + if(obj && (!obj.city && !obj.district)){ + development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata)) + } + } + }) + } + + def addDevAddres1(){ + def rets = [] + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\企业园区地址-新.xlsx",{d-> + if(d['O'] && d['P']=="正常" && d['U']){ + def obj = development_zone_CONN.findOne(new BasicDBObject("name", d['U'])) + if(obj){ + d['AB'] = "存在" + def addr_alias = obj.addr_alias + if(!addr_alias){ + addr_alias = [] + } + def old_count = addr_alias.size() + if(!addr_alias.contains(d['M']+d['N'])){ + addr_alias << d['M']+d['N'] + } + if(old_count != addr_alias.size()){ + def udata = new BasicDBObject() + udata.put("addr_alias", addr_alias) + development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata)) + } + }else{ + rets << d + println "匹配:"+d['U']+" 原始:"+d['H'] + } + } + },0) +// ExcelUtils.write("D:\\数据\\数据建设\\待新增开发区.xlsx",rets, "新增") + } + + def addDevAddres2(){ + + def rets = [] + ExcelUtils.modify("D:\\数据\\数据建设\\待新增国家级以上开发区.xlsx",{d-> + if((d['R'] && d['R'].toString()=="正常") || (d['P'] && d['P'].toString()=="正常")){ + def obj = null + if(d['U']){ + try{ + obj = development_zone_CONN.findOne(new BasicDBObject("name", d['U'])) + }catch(Exception e){ + obj = development_zone_CONN.findOne(new BasicDBObject("name", d['U'])) + } + } + + if(!obj){ + try{ + obj = development_zone_CONN.findOne(new BasicDBObject("name", d['H'])) + }catch(Exception e){ + obj = development_zone_CONN.findOne(new BasicDBObject("name", d['H'])) + } + } + if(obj){ + d['AB'] = "存在" + def addr_alias = obj.addr_alias + if(!addr_alias){ + addr_alias = [] + } + def old_count = addr_alias.size() + + if(d['R'] && d['R']=="正常" && !addr_alias.contains(d['Y']+d['Z']) && d['X'] && d['F'] && d['X']==d['F']){ + addr_alias << d['Y']+d['Z'] + } + + if(d['P'] && d['P']=='正常' && !addr_alias.contains(d['M']+d['N']) && d['F'] && d['L'] && d['L']==d['F']){ + addr_alias << d['M']+d['N'] + } + + if(old_count != addr_alias.size()){ + def udata = new BasicDBObject() + udata.put("addr_alias", addr_alias) + development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata)) + } + }else{ +// rets << d + println "匹配:"+d['U']+" 原始:"+d['H'] + } + } + }, 0) + +// ExcelUtils.write("D:\\数据\\数据建设\\待新增开发区.xlsx",rets, "新增") + } + + def stat(){ + def dbcursor = development_zone_CONN.find(new BasicDBObject()) + def total = 0 + for(def d : dbcursor){ + if(d.addr_alias){ + total = total + d.addr_alias.size() + println d.addr_alias.size() + } + } + println total + } + + def addNewDev(){ + ExcelUtils.modify("D:\\数据\\数据建设\\待新增国家级以上开发区.xlsx",{d-> + def udata = new BasicDBObject() + udata.address = d['M']+d['N'] + udata.create_time = new Date() + if(d['G']){ + if(d['G'].contains("国家级")){ + udata.level = "国家级" + }else if(d['G'].contains("省级")){ + udata.level = "省级" + } + } + + udata.name = d['H'] + udata.province = d['J'] + udata.city = d['K'] + udata.district = d['L'] + udata.type = d['G'] + + + def obj = development_zone_CONN.findOne(new BasicDBObject("name", udata.name)) + if(!obj){ + development_zone_CONN.save(udata) + }else{ + println udata.name +" 已存在" + } + }) + } + + def createDevMapping(){ + def dbcursor = development_zone_CONN.find(new BasicDBObject()) + for(def d : dbcursor){ + def formatAddress = null + if(d.address){ + formatAddress = AddressUtil.formatAddress(d.city, d.address) + } + if(formatAddress && !formatAddress.endsWith("号") && !formatAddress.endsWith("弄")){ + formatAddress = null + } + def addr_alias = [] + if(formatAddress){ + addr_alias << formatAddress + } + if(d.addr_alias){ + addr_alias = addr_alias + d.addr_alias + addr_alias.unique() + } + if(addr_alias){ + addr_alias.each{addr-> + def udata = new BasicDBObject() + udata.put("province", d.province) + udata.put("city", d.city) + udata.put("district", d.district) + udata.put("name",d.name) + udata.put("type", "地址") + if(d.township){ + udata.put("street", d.township) + } + udata.put("address", addr) + DEV_MAPPING_CONN.save(udata) + } + } + } + } + + def deleteAddr(){ + ExcelUtils.modify("D:\\数据\\数据建设\\删除开发区地址.xlsx",{d-> + DEV_MAPPING_CONN.remove(new BasicDBObject("_id", new ObjectId(d['A']))) + }); + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictAliasProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictAliasProcess.groovy new file mode 100644 index 000000000..289e6f68f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictAliasProcess.groovy @@ -0,0 +1,45 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import org.bson.types.ObjectId + +class DistrictAliasProcess { + + static main(args) { + def task = new DistrictAliasProcess(); + task.run() + } + + def run(){ + ExcelUtils.modify("D:\\数据\\建行\\房产估值\\名称在映射别名中的小区(2).xlsx",{ d-> + if(d['H']=="是"){ + def udata = new BasicDBObject() + udata.put("status", 1) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata)) + }else if(d['H']=="否"){ + def aliasDist = d['F'] + def arrs = aliasDist.split("-") + def obj = CommonDatautils.findDistrict(d['B'], arrs[0], arrs[1],"district") + if(obj){ + def alias = obj.alias + def mapping_alias = obj.mapping_alias + def udata = new BasicDBObject() + if(alias && alias.contains(d['D'])){ + alias.remove(d['D']) + udata.put("alias", alias) + } + if(mapping_alias && mapping_alias.contains(d['D'])){ + mapping_alias.remove(d['D']) + udata.put("mapping_alias", mapping_alias) + } + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata)) + } + } + }) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy index c5debfdd2..ea2bd7f20 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy @@ -30,7 +30,7 @@ class TownMiddleProcess { DBCollection REGIONPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") DBCollection TOWNPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline") - def fullCityMap //= CommonDatautils.queryFullNameCity() + def fullCityMap = CommonDatautils.queryFullNameCity() static main(args) { def task = new TownMiddleProcess(); @@ -48,9 +48,10 @@ class TownMiddleProcess { //修改乡镇街道别名 // updateTownAlias() //中间表同步到100 -// synchTown() + synchTown() //添加地级市标志 - addCityFlag() +// addCityFlag() +// addPinyin() } @@ -171,16 +172,33 @@ class TownMiddleProcess { } } //添加别名 - def regionMap = CommonDatautils.getAllCityRegions() + def regionMap = CommonDatautils.getAllCityFullRegions() lastRets.each {o-> + if(o.district.contains("开发区")){ + println "ssss" + } def city = RegionData.formatCity(o.city) def district = RegionData.formatRegion(city, o.district) if(city && district){ def cMap = regionMap.get(city) if(cMap){ - def regions = cMap.get(district) - o.put("alias", regions) + def regionInfoMap = cMap.get(district) + if(regionInfoMap){ + def regions = regionInfoMap.get("pattern") + o.put("alias", regions) + o.put("mzb_district_name", regionInfoMap.full_region) + if(!o.mzb_district_code && regionInfoMap.full_region){ + def mzb_val_str = mzbMaps.get(o.province+"|@|"+o.city+"|@|"+regionInfoMap.full_region) + if(mzb_val_str){ + def mzb_vals = mzb_val_str.split("\\|@\\|") + o.put("mzb_province_code", mzb_vals[0]) + o.put("mzb_city_code", mzb_vals[1]) + o.put("mzb_district_code", mzb_vals[2]) + } + } + } } + }else{ println o.province+" "+o.city+" "+ o.district +" 房价网无此城市" } @@ -719,4 +737,18 @@ class TownMiddleProcess { } },1); } + + def addPinyin(){ + def q = new BasicDBObject("type", "乡镇街道") + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(2000) + for(def d : dbcursor){ + if(d.street){ + def street_pinyin = com.fangjia.core.util.DataUtil.hanyuQuanpin(d.street, " ") + def udata = new BasicDBObject() + udata.put("street_pinyin", street_pinyin) + ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata)) + } + } + + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index a664aefa9..0afe56328 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -354,7 +354,7 @@ class FangjiaClientUtils { def content = null try { address = address.replace("{","") - def url = "http://192.168.0.19:31080/FangJiaAddressService/structureAndComplete?address="+URLEncoder.encode(address,"UTF-8")+"&complete=true&addressAssessment=true" + def url = "http://192.168.0.19:31080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=true&addressAssessment=true" if(city){ url = url +"&city="+URLEncoder.encode(city,"UTF-8") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommbuildtypeBatch.groovy new file mode 100644 index 000000000..423cbd516 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommbuildtypeBatch.groovy @@ -0,0 +1,137 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import groovy.sql.Sql + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 房价网内部模型估值项目 + * 小区建筑类型数据同步 + */ +@BatchOwner(Watcher.GaoJuan) +class FjwBigdataCommbuildtypeBatch extends BaseBatch{ + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + + //235为房价网内部计算均价的库 + def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified", + "postgres", "Goojia123456", "org.postgresql.Driver") + + static main(args) { + runBatch(new FjwBigdataCommbuildtypeBatch(), args); + } + + @Override + public void process(Object args) { + def citys = CcbCommonUtils.getRunningCitys(args) + def rets = [] + def cityCodeDatas = CcbCommonUtils.findCityCodeValue() + + for(def city : citys){ + def distIds = PgsqlCommUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city)) + + def rows = createDatas(cityCodeDatas.get(city), distIds) + rets.addAll(rows) + println city +" "+rows.size() + } + + delPgData() + saveToPgSql(rets) + } + + def createDatas(cityCode, dists){ + def rets = Collections.synchronizedList(new ArrayList()); + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(dists.size()) + for(def i=0;i + for(def r : rows) { + index++ + println index + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14]) + } + } + println "end: "+new Date() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommunityBatch.groovy new file mode 100644 index 000000000..e595ca91a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommunityBatch.groovy @@ -0,0 +1,324 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.core.util.DataUtil +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient +import groovy.sql.Sql + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Pattern + +/** + * 房价网内部估值项目使用 + * 小区数据同步 + */ + +@BatchOwner(Watcher.GaoJuan) +class FjwBigdataCommunityBatch extends BaseBatch{ + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) + + //235为房价网内部计算均价的库 + def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified", + "postgres", "Goojia123456", "org.postgresql.Driver") + + static main(args) { + runBatch(new FjwBigdataCommunityBatch(), args); + } + + @Override + public void process(Object args) { + def citys = CcbCommonUtils.getRunningCitys(args) + def rows = [] + + def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途", + //"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司", + //"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"] + + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def index = 0 + def today = new Date().getDate() + + println "starting 。。。。。。。。。" + for(def city : citys){ + index ++ + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], "all") + def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用 + + def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) + rets = addDeleteDist(dists, ccbMaps, rets) + rows.addAll(rets) + println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size() + } + + if(today > 10){ + delPgData() + saveToPgSql(rows) + } + } + + def addDeleteDist(dists, ccbMaps, rets){ + def fangjiaDists = [] + for(def d : dists){ + fangjiaDists << d._id.toString() + } + println "房价网小区:"+fangjiaDists.size()+" 建行小区:"+rets.size() + def ccbDists = ccbMaps.keySet() + for(def ccbId : ccbDists){ + if(!fangjiaDists.contains(ccbId)){ + def datas = ccbMaps.get(ccbId) + datas[datas.size()-1] = 1 + rets << datas + } + } + println "添加删除状态小区:"+rets.size() + return rets + } + + def createCityDatas(def cityCodeDataMap, def distCodeMap,def dists){ + //def rets = Collections.synchronizedList(new ArrayList()); + def rets = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(dists.size()) + for(def i=0;i + for(def r : rows) { + try{ + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], + r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38]) + ps.toString() + }catch(Exception e){ + e.printStackTrace() + } + } + } + + println "end: "+new Date() + } + + def saveToPgSql2(rows){ + def newRows = [] + for(def r : rows) { + def newR = [] + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ +// r[i] = null + newR << null + }else{ + newR << r[i] + } + } + newR << null + newR << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28" + newR << null + newR << '2999-12-31' + newR << null + newR << null + newR << null + + newRows << newR + } + + println "start: "+new Date() + def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + def index = 0 + sqlInstance.withBatch(50000, sql){ ps-> + for(def r : newRows) { + index++ + try{ + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], + r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38]) + index + }catch(Exception e){ + e.printStackTrace() + } + } + } + + println "end: "+new Date() + } + + + def readTxt(def fileName){ + File file = new File(fileName); + def rows = []; + + file.eachLine{ + def row = it.split("\\|@\\|") + rows << row + } + return rows; + } + + def updateVersion(def time){ + def tables = ["t0831_website_config_gj_h","t0831_bigdata_district_gj_h","t0831_evaluate_rate_gj_h","t0831_bigdata_poi_gj_h","t0831_guided_price_gj_h","t0831_community_staging_gj_h","t0831_community_guide_price_gj_h"] + for(def tableName : tables){ + def sql = "update base."+tableName+" set p9_start_date='"+time+"'" + sqlInstance.executeUpdate(sql) + } + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meishubao/MsbPocTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meishubao/MsbPocTest.groovy index f517a80b8..1c67e8706 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meishubao/MsbPocTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meishubao/MsbPocTest.groovy @@ -13,7 +13,7 @@ class MsbPocTest { task.run(); } def run(){ -// def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(小区、城市均价).xlsx" + def inputfile = "D:\\数据\\建行\\房产估值\\202311广州.xlsx" // def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(匹配V2).xlsx"//美术宝测试数据(匹配).xlsx // standard(inputfile) // match(inputfile) @@ -24,7 +24,7 @@ class MsbPocTest { // cityPrice() // mergeMatchPrice("D:\\数据\\美术宝\\美术宝-新接口价格.xlsx") - addCCBPrice("C:\\Users\\gaojuan\\Desktop\\合肥科技农商行测试.xlsx") + addCCBPrice(inputfile) } def cityPrice(){ @@ -111,23 +111,18 @@ class MsbPocTest { def match(file){ ExcelUtils.multiThreadingModify(file, { row -> - if(!row['J'] && row['F'] && row['P']=="没匹配" ){ - def city = row['C'] - def region = row['D'] - def address = row['G'] //row['G']//row['F'] + if(!row['N']){ + def city = row['A'] + def region = row['B'] + def address = row['C'] //row['G']//row['F'] println "[match] city:"+city+" region:"+region+" address:"+address def jsonObject = FangjiaClientUtils.newMatch(city, address, null, false) - if(jsonObject != null && jsonObject.credit>0) { - row['J'] = jsonObject.get("id") - row['K'] = jsonObject.get("city") - row['L'] = jsonObject.get("district") - row['M'] = jsonObject.get("name") - def type = jsonObject.get("propertyType") - if(type && type!='null'){ - row['N'] = type - } - row['O'] = jsonObject.get("credit") + if(jsonObject != null) { + row['N'] = jsonObject.get("id") + row['O'] = jsonObject.get("city") + row['P'] = jsonObject.get("district") + row['Q'] = jsonObject.get("name") } } }) @@ -174,19 +169,22 @@ class MsbPocTest { def addCCBPrice(path){ def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() ExcelUtils.multiThreadingModify(path, { row -> - if(!row['M']){ - def dist = CommonDatautils.findDistrict(row['E'], row['H'], row['I'], "district") - if(dist){ - def d = CommonDatautils.findDistrictById(dist._id.toString()) + if(!row['H']){ +// def dist = CommonDatautils.findDistrict(row['E'], row['H'], row['I'], "district") +// if(dist){ + if(row['D'] instanceof String){ + def d = CommonDatautils.findDistrictById(row['D']) if(d){ def cityId = cityCodeDataMap.get(d.city) - def map = CcbCommonUtils.findAvgPrice(dist._id.toString(), cityId) + def map = CcbCommonUtils.findAvgPrice(d._id.toString(), cityId) if(map){ - row['M'] = map.avg_price + row['H'] = map.avg_price + row['I'] = map.calc_type } - } } + +// } } }) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy index 7b7b54d0a..81977584c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy @@ -33,7 +33,7 @@ class AddPriceTest { task.run(); } def run(){ - def filePath = "D:\\数据\\建行\\房产估值\\新疆3月待优化小区 - 副本.xlsx" + def filePath = "D:\\数据\\建行\\房产估值\\指导价\\补充154价格.xlsx" // addGXDPrice(filePath) // addCcbPrice(filePath) // addThirdPrice(filePath) @@ -256,14 +256,16 @@ class AddPriceTest { def addFjwPrice(path){ ExcelUtils.multiThreadingModify(path,{d-> - if(!d['P']){ - def dist = CommonDatautils.findDistrictById(d['G']) - if(dist){ + if(!d['F']){ + if(d['E'] instanceof String){ + def dist = CommonDatautils.findDistrictById(d['E']) + if(dist){ // d['N'] = dist.region - def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name) - if(dbo){ - d['O'] = dbo.avg_price + def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name) + if(dbo){ + d['F'] = dbo.avg_price // d['J'] = dbo.avg_price_2b + } } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy index 9801c3588..924db2def 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy @@ -27,8 +27,8 @@ class QiliangTest { // task.run() def data = [ - 'address': '浙江省湖州市德清县莫干山开发区中小工业园区', - 'type': '2' + 'address': '上海市奉贤区星火开发区莲塘路251号8幢', + 'type': '1' //1:对公 0:对私 ] def s = task.getProxyResult(data); print(s); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy index 50882747b..7e85c66ad 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy @@ -414,7 +414,7 @@ public class ExcelUtils{ def data = ExcelUtils.read(fileName, srcSheetIndex, 0) def body = data.subList(1, data.size()) - def nThread = Runtime.getRuntime().availableProcessors() * 4 + def nThread = Runtime.getRuntime().availableProcessors() * 5 println nThread ExecutorService executorService = Executors.newFixedThreadPool(nThread) final AtomicInteger counter = new AtomicInteger(0)