From 5ef49f615da622a5510d1eb49bbd29c7a07ed6f2 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 15 May 2024 16:41:40 +0800 Subject: [PATCH 1/5] update --- .../ccbevaluation/AddThirdPartyPrice.groovy | 2 +- .../ccbevaluation/BigdataBuildingBatch.groovy | 2 +- .../BigdataCommunityBatch.groovy | 2 +- .../ccbevaluation/BigdataDealBatch.groovy | 2 +- .../ccbevaluation/BigdataForsaleBatch.groovy | 2 +- .../ccbevaluation/CcbCommonUtils.groovy | 15 +- .../DistrictGuidePriceBatch.groovy | 2 +- .../ccbevaluation/GuiDingPriceBatch.groovy | 4 +- .../ccbevaluation/StagingDistrictBatch.groovy | 2 +- .../fangjia/script/ccbevaluation/Test.groovy | 17 +- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 2 +- .../ronghe/BigdataCitypriceBatch.groovy | 2 +- .../script/commData/CommonDatautils.groovy | 29 +- .../script/gj/data_handle/DevProcess.groovy | 353 +++++++++++++++++- .../data_handle/DistrictAliasProcess.groovy | 45 +++ .../gj/data_handle/TownMiddleProcess.groovy | 44 ++- .../gj/district/FangjiaClientUtils.groovy | 2 +- .../FjwBigdataCommbuildtypeBatch.groovy | 137 +++++++ .../FjwBigdataCommunityBatch.groovy | 324 ++++++++++++++++ .../script/gj/meishubao/MsbPocTest.groovy | 44 ++- .../fangjia/script/gj/poc/AddPriceTest.groovy | 16 +- .../script/gj/village/QiliangTest.groovy | 4 +- .../com/fangjia/script/util/ExcelUtils.groovy | 2 +- 23 files changed, 988 insertions(+), 66 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictAliasProcess.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommbuildtypeBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommunityBatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPartyPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPartyPrice.groovy index ca14ea514..e0da72624 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPartyPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPartyPrice.groovy @@ -23,7 +23,7 @@ class AddThirdPartyPrice { def cityCodeDatas = CcbCommonUtils.findCityCodeValue() for(def c : citys){ println c +" "+cityCodeDatas.get(c) - def dists = CcbCommonUtils.findAllDistrictByCity(c, "district", ["city", "region", "name","name_cat", "estate_type2","lng"]) + def dists = CcbCommonUtils.findAllDistrictByCity(c, "district", ["city", "region", "name","name_cat", "estate_type2","lng"], null) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(c) def rets = createDate(dists, distCodeMap) println dists.size() +" "+rets.size() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy index 431972b91..2eda22bba 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy @@ -47,7 +47,7 @@ class BigdataBuildingBatch extends BaseBatch{ def getDistrictMap(def city){ - def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias","build_type","build_structure"]); + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias","build_type","build_structure"], null); def distMap = [:] for(def d : districts){ def alias = d.alias; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 0ea40b8c2..47159ee0e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -58,7 +58,7 @@ class BigdataCommunityBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"]) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], null) def ccbMaps = null diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 05a5447e8..a45e49793 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -529,7 +529,7 @@ class BigdataDealBatch extends BaseBatch{ def getCityData(def city, def statMap, def ppeDistsMap){ - def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"]) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"], null) println city +" 小区量:"+dists.size() def tradesList = Collections.synchronizedList(new ArrayList()); def totalCount = new AtomicLong() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 8b444eaba..e329c8732 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -97,7 +97,7 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], null); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219); exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, startDate.toInteger(), endDate.toInteger(), houseIdsfilter); //测试通过匹配接口配房源效果 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 65f84ab77..a1f03acd8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -428,7 +428,7 @@ class CcbCommonUtils { return rows } - public static def findAllDistrictByCity(def city, def cat, def fields){ + public static def findAllDistrictByCity(def city, def cat, def fields, def kind){ DBObject q = new BasicDBObject(); q.put("status",0) @@ -445,14 +445,13 @@ class CcbCommonUtils { keys.put(field, 1); } } -// def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000) def dbsour = SEAWEED_MONGO_COLL.find(q,keys).batchSize(2000) println "[${city}] 原始:"+dbsour.count() def rets = [] def ids = [] for(def d : dbsour){ - if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2) && !d.name.contains("洋房") && !d.name.contains("公房")){ + if(kind){ if(d.address){ d.address = d.address.replaceAll("\r|\n|\\s", ""); } @@ -460,6 +459,16 @@ class CcbCommonUtils { ids << d._id.toString() rets << d } + }else{ + if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2) && !d.name.contains("洋房") && !d.name.contains("公房")){ + if(d.address){ + d.address = d.address.replaceAll("\r|\n|\\s", ""); + } + if(!ids.contains(d._id.toString())){ + ids << d._id.toString() + rets << d + } + } } } println "[${city}] 小区量:"+rets.size() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy index ed5ed5f03..fdf28955b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy @@ -41,7 +41,7 @@ class DistrictGuidePriceBatch extends BaseBatch{ def getDistrictMap(def city){ - def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]); + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"], null); def distMap = [:] for(def d : districts){ distMap.put(d.region+"##"+d.name, d) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 99bed78ed..9ad3f9a31 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2024-03-27", "2024-04-26") + def prices = CcbCommonUtils.findPersonPrice(city, "2024-04-26", "2024-05-10") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) @@ -207,7 +207,7 @@ class GuiDingPriceBatch extends BaseBatch{ } def getDistrictMap(def city){ - def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]); + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"], null); def distMap = [:] for(def d : districts){ distMap.put(d.region+"##"+d.name, d) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy index cae04cf55..82d2c96c0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy @@ -31,7 +31,7 @@ class StagingDistrictBatch extends BaseBatch{ def citys = CcbCommonUtils.getRunningCitys(args) def rows = [] for(def city : citys){ - def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","lng","master_district_name", "c_date", "m_date"]) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","lng","master_district_name", "c_date", "m_date"], null) def fenQiMap = getFenQiMap(dists); def rets = createCityData(dists, fenQiMap) rows.addAll(rets) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 371e260ea..5234f7fd4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -1289,10 +1289,10 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ -def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240425--人工指导价.xls" +def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240508--人工指导价.xls" ExcelUtils.multiThreadingModify(fileName, {d-> - if(d['C']){ + if(d['J']){ // def region = RegionData.formatRegion(d['B'], d['C']) // if(region){ // def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district") @@ -1301,12 +1301,15 @@ ExcelUtils.multiThreadingModify(fileName, {d-> // d['H'] = dist._id.toString() // } // } - if(!d['L']){ - def dist = CommonDatautils.findDistrictById(d['C']) + if(!d['A']){ + def dist = CommonDatautils.findDistrictById(d['J']) if(dist){ - d['L'] = dist.region - d['M'] = dist.name - d['N'] = dist.status + d['A'] = dist.city + d['B'] = dist.region + d['C'] = dist.name + d['K'] = dist.estate_type2 + d['L'] = dist.status + d['M'] = dist.name_cat } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 8483a84a0..b0756a439 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -71,7 +71,7 @@ class ThirdPartyPriceBatch extends BaseBatch { def statMap = getDistrictStatByCity(city, weekend) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"]) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"], null) def thirdPriceMap = findThirdPartyPrice(city, statMap) //def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap)//老第三方均价表(现不用) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index 5a38c237e..479813af4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -207,7 +207,7 @@ class BigdataCitypriceBatch extends BaseBatch{ def allLastMonthMap = [:] def allLastYearMap = [:] for(def city : citys) { - def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias"]); + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias"], null); allDistricts.addAll(districts) def priceMap = CommonDatautils.queryDistrictStatByDate(city, startDate, endDate) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 233a8e0f7..fde3609ec 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1720,7 +1720,34 @@ class CommonDatautils { } return regionMap; } - + + + public static def getAllCityFullRegions(){ + DBObject queryCondition = new BasicDBObject() + queryCondition.put("category","region"); + //queryCondition.put("show",1); + def dbCursor = REGION_BLOCK_COLL.find(queryCondition) + def regionMap = [:]; + for(DBObject obj : dbCursor){ + def city = obj.get("city"); + def region = obj.get("name"); + def pattern = obj.get("pattern"); + if(region.indexOf("周边")==-1 && region.indexOf("其他")==-1) { + def r_map = regionMap.get(city); + if(!r_map){ + r_map = [:] + regionMap.put(city, r_map); + } + + if(!pattern){ + pattern = []; + pattern << region; + } + r_map.put(region, ["pattern":pattern, "full_region":obj.get("org_name")]); + } + } + return regionMap; + } public static def getRegionsByCity(def city){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index 90d781a2f..342e946fb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -1,25 +1,44 @@ package com.fangjia.script.gj.data_handle import com.fangjia.core.RegionData +import com.fangjia.core.util.GeoUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.linxsh.util.AddressUtil import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBObject +import com.mongodb.Bytes import com.mongodb.DBCollection import net.sf.json.JSONObject import org.apache.poi.ss.util.RegionUtil +import org.bson.types.ObjectId class DevProcess { def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL) + DBCollection DEV_MAPPING_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL) + static main(args) { def task = new DevProcess(); // task.run() - task.parseCityAndDist() +// task.parseCityAndDist() +// task.devSplitRoad() +// task.addFlag() +// task.matchDev() + +// task.calAddrDistince() +// task.updateCityAndDist() +// task.addDevAddres1() //按添加地址中包含开发区名称的地址 +// task.addDevAddres2()//添加相差2公里范围内的地址 + +// task.stat() +// task.addNewDev() +// task.createDevMapping() + task.deleteAddr() } def parseCityAndDist(){ @@ -131,10 +150,336 @@ class DevProcess { }else{ println udata.name +" 已存在" } - - } - } } + + def devSplitRoad(){ + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\企业园区地址-新.xlsx",{d-> + /*if(d['C'] && !d['V']){ + def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['E'], d['C']) + def street = jsonObject.get("street") + def streetNumber = jsonObject.get("streetNumber") + if(jsonObject.get("province") && jsonObject.get("province")!="null"){ + d['V'] = jsonObject.get("province") + } + + if(jsonObject.get("city") && jsonObject.get("city")!="null"){ + d['W'] = jsonObject.get("city") + } + + if(jsonObject.get("district") && jsonObject.get("district")!="null"){ + d['X'] = jsonObject.get("district") + } + + if(jsonObject && street && streetNumber && street!="null"){ + d['Y'] = jsonObject.get("street") + if(streetNumber!='null'){ + d['Z'] = jsonObject.get("streetNumber") + } + } + }*/ + if(d['D'] && !d['M'] && d['P'] && d['P'].toString().toDouble().toInteger()==1){ + def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['E'], d['D']) + def street = jsonObject.get("street") + def streetNumber = jsonObject.get("streetNumber") + if(jsonObject.get("province") && jsonObject.get("province")!="null"){ + d['J'] = jsonObject.get("province") + } + + if(jsonObject.get("city") && jsonObject.get("city")!="null"){ + d['K'] = jsonObject.get("city") + } + + if(jsonObject.get("district") && jsonObject.get("district")!="null"){ + d['L'] = jsonObject.get("district") + } + + if(jsonObject && street && streetNumber && street!="null"){ + d['M'] = jsonObject.get("street") + if(streetNumber!='null'){ + d['N'] = jsonObject.get("streetNumber") + } + } + } + },0) + } + + def addFlag(){ + ExcelUtils.modify("C:\\Users\\gaojuan\\Downloads\\企业园区地址.xlsx",{d-> + if(d['H'] && d['D']){ + if(d['D'].contains(d['H'])){ + d['O'] = "是" + }else{ + def newName = null + if(d['H'].toString().startsWith(d['I'])){ + newName = d['H'].toString().substring(d['I'].toString().length()) + }else if(d['H'].toString().startsWith(d['I']+"省")){ + newName = d['H'].toString().substring(d['I'].toString().length()+1) + } + if(newName && d['D'].contains(newName)){ + d['O'] = "是2" + } + } + } + },1); + } + + def matchDev(){ + ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Downloads\\企业园区地址.xlsx",{d-> + if(d['H']){ + def q = new BasicDBObject() + q.put("name", d['H']) + def obj = development_zone_CONN.findOne(q) + if(obj){ + d['S'] = obj._id.toString() + d['T'] = obj.name + } + } + },1); + } + + def calAddrDistince(){ + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\企业园区地址-新.xlsx",{d-> + if(d['M'] && d['N'] && d['Y'] && d['Z']){ + def addr1 = getFullAddress(d['J'], d['K'],d['L'],d['M'] , d['N']) + def addr2 = getFullAddress(d['V'], d['W'], d['X'], d['Y'] , d['Z']) + + def json1 = GaoDeApiUtils.geo(d['K'], addr1) + def json2 = GaoDeApiUtils.geo(d['W'], addr2) + def arrs1 = getLocation(json1) + def arrs2 = getLocation(json2) + if(arrs1 && arrs2){ + def distance = GeoUtils.calculateDistance(arrs1[0].toString().toDouble(), arrs1[1].toString().toDouble(), arrs2[0].toString().toDouble() , arrs2[1].toString().toDouble() ) + d['AA'] = distance + } + } + },0); + } + + def getFullAddress(province, city, region, street, streetName){ + def addr = "" + if(addr){ + addr = province + } + if(city){ + addr = addr + city + } + if(region){ + addr = addr + region + } + addr = addr + street + streetName + return addr + } + def getLocation(json1){ + def arrs1 = null + if(json1 && json1.status=="1") { + def ret = json1.geocodes + if (ret) { + def str = ret.get(0).location + if(str){ + arrs1 = str.split(",") + } + } + } + return arrs1 + } + + def updateCityAndDist(){ + ExcelUtils.modify("D:\\数据\\数据建设\\等级开发区 - 汇总.xlsx",{d-> + if(d['L']){ + def udata = new BasicDBObject() + if(d['L']=="采用拆分结果"){ + if(d["I"]){ + udata.put("province", d["I"]) + } + if(d["J"]){ + udata.put("city", d["J"]) + } + if(d["K"]){ + udata.put("district", d["K"]) + } + }else if(d['L']=="采用地图结果"){ + if(d["H"]){ + udata.put("province", d["H"]) + } + if(d["E"]){ + udata.put("city", d["E"]) + } + if(d["F"]){ + udata.put("district", d["F"]) + } + } + + def obj = development_zone_CONN.findOne(new BasicDBObject("name", d['C'])) + if(obj && (!obj.city && !obj.district)){ + development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata)) + } + } + }) + } + + def addDevAddres1(){ + def rets = [] + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\企业园区地址-新.xlsx",{d-> + if(d['O'] && d['P']=="正常" && d['U']){ + def obj = development_zone_CONN.findOne(new BasicDBObject("name", d['U'])) + if(obj){ + d['AB'] = "存在" + def addr_alias = obj.addr_alias + if(!addr_alias){ + addr_alias = [] + } + def old_count = addr_alias.size() + if(!addr_alias.contains(d['M']+d['N'])){ + addr_alias << d['M']+d['N'] + } + if(old_count != addr_alias.size()){ + def udata = new BasicDBObject() + udata.put("addr_alias", addr_alias) + development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata)) + } + }else{ + rets << d + println "匹配:"+d['U']+" 原始:"+d['H'] + } + } + },0) +// ExcelUtils.write("D:\\数据\\数据建设\\待新增开发区.xlsx",rets, "新增") + } + + def addDevAddres2(){ + + def rets = [] + ExcelUtils.modify("D:\\数据\\数据建设\\待新增国家级以上开发区.xlsx",{d-> + if((d['R'] && d['R'].toString()=="正常") || (d['P'] && d['P'].toString()=="正常")){ + def obj = null + if(d['U']){ + try{ + obj = development_zone_CONN.findOne(new BasicDBObject("name", d['U'])) + }catch(Exception e){ + obj = development_zone_CONN.findOne(new BasicDBObject("name", d['U'])) + } + } + + if(!obj){ + try{ + obj = development_zone_CONN.findOne(new BasicDBObject("name", d['H'])) + }catch(Exception e){ + obj = development_zone_CONN.findOne(new BasicDBObject("name", d['H'])) + } + } + if(obj){ + d['AB'] = "存在" + def addr_alias = obj.addr_alias + if(!addr_alias){ + addr_alias = [] + } + def old_count = addr_alias.size() + + if(d['R'] && d['R']=="正常" && !addr_alias.contains(d['Y']+d['Z']) && d['X'] && d['F'] && d['X']==d['F']){ + addr_alias << d['Y']+d['Z'] + } + + if(d['P'] && d['P']=='正常' && !addr_alias.contains(d['M']+d['N']) && d['F'] && d['L'] && d['L']==d['F']){ + addr_alias << d['M']+d['N'] + } + + if(old_count != addr_alias.size()){ + def udata = new BasicDBObject() + udata.put("addr_alias", addr_alias) + development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata)) + } + }else{ +// rets << d + println "匹配:"+d['U']+" 原始:"+d['H'] + } + } + }, 0) + +// ExcelUtils.write("D:\\数据\\数据建设\\待新增开发区.xlsx",rets, "新增") + } + + def stat(){ + def dbcursor = development_zone_CONN.find(new BasicDBObject()) + def total = 0 + for(def d : dbcursor){ + if(d.addr_alias){ + total = total + d.addr_alias.size() + println d.addr_alias.size() + } + } + println total + } + + def addNewDev(){ + ExcelUtils.modify("D:\\数据\\数据建设\\待新增国家级以上开发区.xlsx",{d-> + def udata = new BasicDBObject() + udata.address = d['M']+d['N'] + udata.create_time = new Date() + if(d['G']){ + if(d['G'].contains("国家级")){ + udata.level = "国家级" + }else if(d['G'].contains("省级")){ + udata.level = "省级" + } + } + + udata.name = d['H'] + udata.province = d['J'] + udata.city = d['K'] + udata.district = d['L'] + udata.type = d['G'] + + + def obj = development_zone_CONN.findOne(new BasicDBObject("name", udata.name)) + if(!obj){ + development_zone_CONN.save(udata) + }else{ + println udata.name +" 已存在" + } + }) + } + + def createDevMapping(){ + def dbcursor = development_zone_CONN.find(new BasicDBObject()) + for(def d : dbcursor){ + def formatAddress = null + if(d.address){ + formatAddress = AddressUtil.formatAddress(d.city, d.address) + } + if(formatAddress && !formatAddress.endsWith("号") && !formatAddress.endsWith("弄")){ + formatAddress = null + } + def addr_alias = [] + if(formatAddress){ + addr_alias << formatAddress + } + if(d.addr_alias){ + addr_alias = addr_alias + d.addr_alias + addr_alias.unique() + } + if(addr_alias){ + addr_alias.each{addr-> + def udata = new BasicDBObject() + udata.put("province", d.province) + udata.put("city", d.city) + udata.put("district", d.district) + udata.put("name",d.name) + udata.put("type", "地址") + if(d.township){ + udata.put("street", d.township) + } + udata.put("address", addr) + DEV_MAPPING_CONN.save(udata) + } + } + } + } + + def deleteAddr(){ + ExcelUtils.modify("D:\\数据\\数据建设\\删除开发区地址.xlsx",{d-> + DEV_MAPPING_CONN.remove(new BasicDBObject("_id", new ObjectId(d['A']))) + }); + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictAliasProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictAliasProcess.groovy new file mode 100644 index 000000000..289e6f68f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictAliasProcess.groovy @@ -0,0 +1,45 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import org.bson.types.ObjectId + +class DistrictAliasProcess { + + static main(args) { + def task = new DistrictAliasProcess(); + task.run() + } + + def run(){ + ExcelUtils.modify("D:\\数据\\建行\\房产估值\\名称在映射别名中的小区(2).xlsx",{ d-> + if(d['H']=="是"){ + def udata = new BasicDBObject() + udata.put("status", 1) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata)) + }else if(d['H']=="否"){ + def aliasDist = d['F'] + def arrs = aliasDist.split("-") + def obj = CommonDatautils.findDistrict(d['B'], arrs[0], arrs[1],"district") + if(obj){ + def alias = obj.alias + def mapping_alias = obj.mapping_alias + def udata = new BasicDBObject() + if(alias && alias.contains(d['D'])){ + alias.remove(d['D']) + udata.put("alias", alias) + } + if(mapping_alias && mapping_alias.contains(d['D'])){ + mapping_alias.remove(d['D']) + udata.put("mapping_alias", mapping_alias) + } + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata)) + } + } + }) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy index c5debfdd2..ea2bd7f20 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy @@ -30,7 +30,7 @@ class TownMiddleProcess { DBCollection REGIONPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") DBCollection TOWNPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline") - def fullCityMap //= CommonDatautils.queryFullNameCity() + def fullCityMap = CommonDatautils.queryFullNameCity() static main(args) { def task = new TownMiddleProcess(); @@ -48,9 +48,10 @@ class TownMiddleProcess { //修改乡镇街道别名 // updateTownAlias() //中间表同步到100 -// synchTown() + synchTown() //添加地级市标志 - addCityFlag() +// addCityFlag() +// addPinyin() } @@ -171,16 +172,33 @@ class TownMiddleProcess { } } //添加别名 - def regionMap = CommonDatautils.getAllCityRegions() + def regionMap = CommonDatautils.getAllCityFullRegions() lastRets.each {o-> + if(o.district.contains("开发区")){ + println "ssss" + } def city = RegionData.formatCity(o.city) def district = RegionData.formatRegion(city, o.district) if(city && district){ def cMap = regionMap.get(city) if(cMap){ - def regions = cMap.get(district) - o.put("alias", regions) + def regionInfoMap = cMap.get(district) + if(regionInfoMap){ + def regions = regionInfoMap.get("pattern") + o.put("alias", regions) + o.put("mzb_district_name", regionInfoMap.full_region) + if(!o.mzb_district_code && regionInfoMap.full_region){ + def mzb_val_str = mzbMaps.get(o.province+"|@|"+o.city+"|@|"+regionInfoMap.full_region) + if(mzb_val_str){ + def mzb_vals = mzb_val_str.split("\\|@\\|") + o.put("mzb_province_code", mzb_vals[0]) + o.put("mzb_city_code", mzb_vals[1]) + o.put("mzb_district_code", mzb_vals[2]) + } + } + } } + }else{ println o.province+" "+o.city+" "+ o.district +" 房价网无此城市" } @@ -719,4 +737,18 @@ class TownMiddleProcess { } },1); } + + def addPinyin(){ + def q = new BasicDBObject("type", "乡镇街道") + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(2000) + for(def d : dbcursor){ + if(d.street){ + def street_pinyin = com.fangjia.core.util.DataUtil.hanyuQuanpin(d.street, " ") + def udata = new BasicDBObject() + udata.put("street_pinyin", street_pinyin) + ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata)) + } + } + + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index a664aefa9..0afe56328 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -354,7 +354,7 @@ class FangjiaClientUtils { def content = null try { address = address.replace("{","") - def url = "http://192.168.0.19:31080/FangJiaAddressService/structureAndComplete?address="+URLEncoder.encode(address,"UTF-8")+"&complete=true&addressAssessment=true" + def url = "http://192.168.0.19:31080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=true&addressAssessment=true" if(city){ url = url +"&city="+URLEncoder.encode(city,"UTF-8") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommbuildtypeBatch.groovy new file mode 100644 index 000000000..423cbd516 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommbuildtypeBatch.groovy @@ -0,0 +1,137 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import groovy.sql.Sql + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 房价网内部模型估值项目 + * 小区建筑类型数据同步 + */ +@BatchOwner(Watcher.GaoJuan) +class FjwBigdataCommbuildtypeBatch extends BaseBatch{ + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + + //235为房价网内部计算均价的库 + def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified", + "postgres", "Goojia123456", "org.postgresql.Driver") + + static main(args) { + runBatch(new FjwBigdataCommbuildtypeBatch(), args); + } + + @Override + public void process(Object args) { + def citys = CcbCommonUtils.getRunningCitys(args) + def rets = [] + def cityCodeDatas = CcbCommonUtils.findCityCodeValue() + + for(def city : citys){ + def distIds = PgsqlCommUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city)) + + def rows = createDatas(cityCodeDatas.get(city), distIds) + rets.addAll(rows) + println city +" "+rows.size() + } + + delPgData() + saveToPgSql(rets) + } + + def createDatas(cityCode, dists){ + def rets = Collections.synchronizedList(new ArrayList()); + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(dists.size()) + for(def i=0;i + for(def r : rows) { + index++ + println index + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14]) + } + } + println "end: "+new Date() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommunityBatch.groovy new file mode 100644 index 000000000..e595ca91a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommunityBatch.groovy @@ -0,0 +1,324 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.core.util.DataUtil +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient +import groovy.sql.Sql + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Pattern + +/** + * 房价网内部估值项目使用 + * 小区数据同步 + */ + +@BatchOwner(Watcher.GaoJuan) +class FjwBigdataCommunityBatch extends BaseBatch{ + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) + + //235为房价网内部计算均价的库 + def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified", + "postgres", "Goojia123456", "org.postgresql.Driver") + + static main(args) { + runBatch(new FjwBigdataCommunityBatch(), args); + } + + @Override + public void process(Object args) { + def citys = CcbCommonUtils.getRunningCitys(args) + def rows = [] + + def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途", + //"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司", + //"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"] + + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def index = 0 + def today = new Date().getDate() + + println "starting 。。。。。。。。。" + for(def city : citys){ + index ++ + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], "all") + def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用 + + def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) + rets = addDeleteDist(dists, ccbMaps, rets) + rows.addAll(rets) + println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size() + } + + if(today > 10){ + delPgData() + saveToPgSql(rows) + } + } + + def addDeleteDist(dists, ccbMaps, rets){ + def fangjiaDists = [] + for(def d : dists){ + fangjiaDists << d._id.toString() + } + println "房价网小区:"+fangjiaDists.size()+" 建行小区:"+rets.size() + def ccbDists = ccbMaps.keySet() + for(def ccbId : ccbDists){ + if(!fangjiaDists.contains(ccbId)){ + def datas = ccbMaps.get(ccbId) + datas[datas.size()-1] = 1 + rets << datas + } + } + println "添加删除状态小区:"+rets.size() + return rets + } + + def createCityDatas(def cityCodeDataMap, def distCodeMap,def dists){ + //def rets = Collections.synchronizedList(new ArrayList()); + def rets = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(dists.size()) + for(def i=0;i + for(def r : rows) { + try{ + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], + r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38]) + ps.toString() + }catch(Exception e){ + e.printStackTrace() + } + } + } + + println "end: "+new Date() + } + + def saveToPgSql2(rows){ + def newRows = [] + for(def r : rows) { + def newR = [] + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ +// r[i] = null + newR << null + }else{ + newR << r[i] + } + } + newR << null + newR << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28" + newR << null + newR << '2999-12-31' + newR << null + newR << null + newR << null + + newRows << newR + } + + println "start: "+new Date() + def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + def index = 0 + sqlInstance.withBatch(50000, sql){ ps-> + for(def r : newRows) { + index++ + try{ + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], + r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38]) + index + }catch(Exception e){ + e.printStackTrace() + } + } + } + + println "end: "+new Date() + } + + + def readTxt(def fileName){ + File file = new File(fileName); + def rows = []; + + file.eachLine{ + def row = it.split("\\|@\\|") + rows << row + } + return rows; + } + + def updateVersion(def time){ + def tables = ["t0831_website_config_gj_h","t0831_bigdata_district_gj_h","t0831_evaluate_rate_gj_h","t0831_bigdata_poi_gj_h","t0831_guided_price_gj_h","t0831_community_staging_gj_h","t0831_community_guide_price_gj_h"] + for(def tableName : tables){ + def sql = "update base."+tableName+" set p9_start_date='"+time+"'" + sqlInstance.executeUpdate(sql) + } + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meishubao/MsbPocTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meishubao/MsbPocTest.groovy index f517a80b8..1c67e8706 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meishubao/MsbPocTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meishubao/MsbPocTest.groovy @@ -13,7 +13,7 @@ class MsbPocTest { task.run(); } def run(){ -// def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(小区、城市均价).xlsx" + def inputfile = "D:\\数据\\建行\\房产估值\\202311广州.xlsx" // def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(匹配V2).xlsx"//美术宝测试数据(匹配).xlsx // standard(inputfile) // match(inputfile) @@ -24,7 +24,7 @@ class MsbPocTest { // cityPrice() // mergeMatchPrice("D:\\数据\\美术宝\\美术宝-新接口价格.xlsx") - addCCBPrice("C:\\Users\\gaojuan\\Desktop\\合肥科技农商行测试.xlsx") + addCCBPrice(inputfile) } def cityPrice(){ @@ -111,23 +111,18 @@ class MsbPocTest { def match(file){ ExcelUtils.multiThreadingModify(file, { row -> - if(!row['J'] && row['F'] && row['P']=="没匹配" ){ - def city = row['C'] - def region = row['D'] - def address = row['G'] //row['G']//row['F'] + if(!row['N']){ + def city = row['A'] + def region = row['B'] + def address = row['C'] //row['G']//row['F'] println "[match] city:"+city+" region:"+region+" address:"+address def jsonObject = FangjiaClientUtils.newMatch(city, address, null, false) - if(jsonObject != null && jsonObject.credit>0) { - row['J'] = jsonObject.get("id") - row['K'] = jsonObject.get("city") - row['L'] = jsonObject.get("district") - row['M'] = jsonObject.get("name") - def type = jsonObject.get("propertyType") - if(type && type!='null'){ - row['N'] = type - } - row['O'] = jsonObject.get("credit") + if(jsonObject != null) { + row['N'] = jsonObject.get("id") + row['O'] = jsonObject.get("city") + row['P'] = jsonObject.get("district") + row['Q'] = jsonObject.get("name") } } }) @@ -174,19 +169,22 @@ class MsbPocTest { def addCCBPrice(path){ def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() ExcelUtils.multiThreadingModify(path, { row -> - if(!row['M']){ - def dist = CommonDatautils.findDistrict(row['E'], row['H'], row['I'], "district") - if(dist){ - def d = CommonDatautils.findDistrictById(dist._id.toString()) + if(!row['H']){ +// def dist = CommonDatautils.findDistrict(row['E'], row['H'], row['I'], "district") +// if(dist){ + if(row['D'] instanceof String){ + def d = CommonDatautils.findDistrictById(row['D']) if(d){ def cityId = cityCodeDataMap.get(d.city) - def map = CcbCommonUtils.findAvgPrice(dist._id.toString(), cityId) + def map = CcbCommonUtils.findAvgPrice(d._id.toString(), cityId) if(map){ - row['M'] = map.avg_price + row['H'] = map.avg_price + row['I'] = map.calc_type } - } } + +// } } }) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy index 7b7b54d0a..81977584c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy @@ -33,7 +33,7 @@ class AddPriceTest { task.run(); } def run(){ - def filePath = "D:\\数据\\建行\\房产估值\\新疆3月待优化小区 - 副本.xlsx" + def filePath = "D:\\数据\\建行\\房产估值\\指导价\\补充154价格.xlsx" // addGXDPrice(filePath) // addCcbPrice(filePath) // addThirdPrice(filePath) @@ -256,14 +256,16 @@ class AddPriceTest { def addFjwPrice(path){ ExcelUtils.multiThreadingModify(path,{d-> - if(!d['P']){ - def dist = CommonDatautils.findDistrictById(d['G']) - if(dist){ + if(!d['F']){ + if(d['E'] instanceof String){ + def dist = CommonDatautils.findDistrictById(d['E']) + if(dist){ // d['N'] = dist.region - def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name) - if(dbo){ - d['O'] = dbo.avg_price + def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name) + if(dbo){ + d['F'] = dbo.avg_price // d['J'] = dbo.avg_price_2b + } } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy index 9801c3588..924db2def 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy @@ -27,8 +27,8 @@ class QiliangTest { // task.run() def data = [ - 'address': '浙江省湖州市德清县莫干山开发区中小工业园区', - 'type': '2' + 'address': '上海市奉贤区星火开发区莲塘路251号8幢', + 'type': '1' //1:对公 0:对私 ] def s = task.getProxyResult(data); print(s); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy index 50882747b..7e85c66ad 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy @@ -414,7 +414,7 @@ public class ExcelUtils{ def data = ExcelUtils.read(fileName, srcSheetIndex, 0) def body = data.subList(1, data.size()) - def nThread = Runtime.getRuntime().availableProcessors() * 4 + def nThread = Runtime.getRuntime().availableProcessors() * 5 println nThread ExecutorService executorService = Executors.newFixedThreadPool(nThread) final AtomicInteger counter = new AtomicInteger(0) From 12bbf3b30d6abdd5a5b8e6ae409294d963858b41 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 15 May 2024 18:05:02 +0800 Subject: [PATCH 2/5] =?UTF-8?q?=E5=B0=8F=E5=8C=BA=E5=90=8C=E6=AD=A5?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/CcbCommonUtils.groovy | 6 +- .../com/fangjia/script/gj/FiledStrand.groovy | 1 + .../FjwBigdataCommbuildtypeBatch.groovy | 2 +- .../FjwBigdataCommunityBatch.groovy | 50 +- .../FjwBigdataForsaleBatch.groovy | 1150 +++++++++++++++++ 5 files changed, 1205 insertions(+), 4 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index a1f03acd8..3409b89b4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -16,7 +16,7 @@ import groovy.sql.Sql class CcbCommonUtils { // -// public static def citys_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"] + public static def citys_fjw_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山"] public static def citys_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"] public static def citys_92 = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"] public static def citys_252 = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"] @@ -69,6 +69,10 @@ class CcbCommonUtils { citys = CcbCommonUtils.citys_252 sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + }else if(countStr=="all"){ + citys = CcbCommonUtils.citys_fjw_all + sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") } }else{ citys = CcbCommonUtils.citys_all diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy index db6f86848..1b11ee16a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy @@ -238,6 +238,7 @@ class FiledStrand { } static def getValue(def val){ + val = val.replace(",","") val = getNumber(val); if(val){ if(val.indexOf(".")>-1 && !val.toString().endsWith(".0")){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommbuildtypeBatch.groovy index 423cbd516..b76f4139a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommbuildtypeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommbuildtypeBatch.groovy @@ -1,4 +1,4 @@ -package com.fangjia.script.ccbevaluation +package com.fangjia.script.gj.fjwPriceBaseData import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommunityBatch.groovy index e595ca91a..4a0358411 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommunityBatch.groovy @@ -1,10 +1,11 @@ -package com.fangjia.script.ccbevaluation +package com.fangjia.script.gj.fjwPriceBaseData import com.fangjia.core.util.DataUtil import com.fangjia.core.util.GeoUtils import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher +import com.fangjia.script.ccbevaluation.CcbCommonUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.ccb.CcbCodeUtils import com.fangjia.script.gj.ccb.FileUtils @@ -41,6 +42,7 @@ class FjwBigdataCommunityBatch extends BaseBatch{ @Override public void process(Object args) { + def citys = CcbCommonUtils.getRunningCitys(args) def rows = [] @@ -57,7 +59,7 @@ class FjwBigdataCommunityBatch extends BaseBatch{ index ++ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], "all") - def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用 + def ccbMaps = findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用 def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) rets = addDeleteDist(dists, ccbMaps, rets) @@ -219,6 +221,50 @@ class FjwBigdataCommunityBatch extends BaseBatch{ return ret; } + + public def findPgCcbDistrict(cityid){ + def sql = "select * from base.bigdata_community where urbn_id='${cityid}'"; + def map = [:] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_admndiv_id + data << r.gujia_comm_id + data << r.gujia_admndiv_nm + data << r.comm_nm + data << r.gujia_dtl_adr + data << r.gujia_bds_ltt + data << r.gujia_bds_lgt + data << r.gujiabdsocpoldtotarea + data << r.gujiabds_bld_tot_area + data << r.gujia_bds_grnrate + data << r.gujia_bds_vol_rate + data << r.gujia_bds_park_num + data << r.gujia_bds_bld_yr_rc + data << r.gujia_bds_hs_use_val + data << r.totunits + data << r.gujia_bds_data_tm_val + data << r.gujiabds_udt_flag_val + data << r.data_udt_tm + data << r.gujia_comm_tp_val + data << r.gujia_comm_fmnm + data << r.gujia_str + data << r.gujia_hsno + data << r.gujiawthr_new_pan_ind + data << r.gujia_comm_intd + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.gujia_src_co + data << r.gujia_vlgtwn_str_mng + data << r.gujia_lgc_admndiv_cd + data << r.gujia_lgc_adiv + data << r.gujia_bds_bds_st + map.put(r.gujia_comm_id, data) + }); + return map + } + + def delPgData(){ sqlInstance.execute("delete from base.bigdata_community") println "base.bigdata_community 数据已删除" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy new file mode 100644 index 000000000..908bdc375 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy @@ -0,0 +1,1150 @@ +package com.fangjia.script.gj.fjwPriceBaseData + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.google.common.hash.* +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.sql.Sql +import org.apache.commons.lang.StringUtils +import org.bson.types.ObjectId +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.nio.charset.StandardCharsets +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Matcher +import java.util.regex.Pattern + + +/** + * 建行模型估值项目 + * 出售房源数据同步 + */ +@BatchOwner(Watcher.GaoJuan) +class FjwBigdataForsaleBatch extends BaseBatch{ + public static EsDataSource dataSource = new EsDataSource() + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" + static { + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + connectionProperties.put("user", "developer") + connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") + dataSource.setProperties(connectionProperties) + } + + static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"] + private static final HashMap> FITMENTS_TYPES_MAPS = new HashMap>() + static{ + FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"]) + FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"]) + FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"]) + FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"]) + } + + public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"]; + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60) + + public static ExecutorService fixedThreadPool2 = Executors.newFixedThreadPool(200) + + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/fangjia?stringtype=unspecified", + "postgres", "Goojia123456", "org.postgresql.Driver") + + + def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"); + def curMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"); + + static main(args) { + runBatch(new FjwBigdataForsaleBatch(), args); + } + + @Override + public void process(Object args) { + //目前查询库中是否已存在,用的是Oracle + def index = 0 + def citys = CcbCommonUtils.getRunningCitys(args) + + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + + + def startDate = lastMonth+"18" + def endDate = curMonth+"19" + println "房源时间:"+startDate+" "+endDate + + def houseIdsfilter = findAllHouseIdsByCity() + + for(def city : citys){ + index++ + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], "all"); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, startDate.toInteger(), endDate.toInteger(), houseIdsfilter); + } + } + + + def statCity(def citys){ + def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\无房源城市.xlsx",0,1) + def houseCitys = [] + for(def d : datas){ + houseCitys << d['C'] + } + for(def city : citys){ + if(!houseCitys.contains(city)){ + def scity = CcbCommonUtils.specialCityMap.get(city) + if(!scity){ + println city + } + } + } + } + + def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate, houseIdsfilter){ + + def retMap = findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate, houseIdsfilter) + println city + "(原)房源总量:"+retMap.size() + + def allDatas = retMap.values() + saveToPgSql(allDatas) + } + + def findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate, houseIdsfilter){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + println city +" 小区量:"+districts.size() + + + + def retMap = new ConcurrentHashMap() + for(def i=0;i60){ + continue; + } + +// if(!seawater.room){ +// continue; +// } + if(!seawater.room){ + seawater.room = null + } + + if(seawater.room){ + if(seawater.room>=11 || seawater.room<1){ + continue; + } + } + + if(!seawater.hall){ + seawater.hall = null + } + if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){ + continue; + } + if(!seawater.toilet){ + seawater.toilet = null + } + if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){ + continue; + } + + + def floor = parseFloor(seawater.floor_info) + if(seawater.height && floor && floor.isNumber()){ + if(floor.toInteger()>seawater.height){ + floor = null + seawater.height = null + } + + } + + if(!floor){ + seawater.floor = null + } + if(floor && floor.toInteger()<1){ + floor = null + } + def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room) + if(vallaFlag){ + continue; + } + + def rowMap = [:]; + if(cat == "rent"){ + rowMap.rent_type = seawater.rent_type + if(seawater.avg_price && seawater.avg_price>1000){ + continue + } + + if(seawater.avg_price && seawater.avg_price<10){ + continue + } + + } + rowMap.houseId = seawater.id + rowMap.city = city + //rowMap.region =seawater.region + //rowMap.address = seawater.address + + + rowMap.area = seawater.area + rowMap.total_price = seawater.total_price + rowMap.avg_price = seawater.avg_price + rowMap.height = seawater.height=="null"?null:seawater.height + if(rowMap.height && rowMap.height<1){ + rowMap.height = null + } + rowMap.floor = floor=="null"?null:floor + rowMap.room = seawater.room=="null"?null:seawater.room + rowMap.hall = seawater.hall=="null"?null:seawater.hall + rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet + if(CcbCommonUtils.rootMap.get(seawater.root)){ + rowMap.root = CcbCommonUtils.rootMap.get(seawater.root) + }else{ + rowMap.root = "99" + } + + def direction = parseDirection(seawater.direction) + rowMap.direction = direction=="null"?null:direction + + def fitment = parseFitment(seawater.fitment) + rowMap.fitment = fitment=="null"?null:fitment + + + + if(seawater.p_date instanceof String){ + seawater.p_date = seawater.p_date.toInteger() + } + if(seawater.p_date<=20200101){ + //seawater.p_date.toString().replace("2019","2020")// + rowMap.p_date = DateUtils.getRandomDate("2020-01-01", "2020-07-31") + rowMap.c_date = rowMap.p_date + rowMap.m_date = rowMap.p_date + }else{ + rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") + rowMap.c_date = DateUtils.getConverDate(seawater.m_date) + rowMap.m_date = DateUtils.getConverDate(seawater.m_date) + } + + rets << rowMap + } + } + + println commId +" "+houses.size()+" last:"+rets.size() + return rets; + } + + + def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, def commId, def cityCodeDataMap){ + + def rets1 = []//findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2) + def rets2 = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1) + rets1.addAll(rets2) +// if(rets1){ +// def newRets = [] +// def cityCode = cityCodeDataMap.get(city) +// def houseIds = getAllHouseIds(commId, cityCode) +// for(def r : rets1){ +// if(!houseIds.contains(r.houseId)){ +// newRets << r +// }else{ +// println r.houseId +" 已存在" +// } +// } +// return newRets +// } +// return []; + return rets1; + } + + def findHousesByPdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){ + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range) + + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + + + if(name.contains("弄小区")){ + def newName = name.replace("弄小区","弄") + if(!alias.contains(newName)){ + alias << newName + } + } + + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + + def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"] + def index = new Random().nextInt(6); + + def rets = [] + def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder); + + if(houses){ + for (int k=0;k60){ + continue; + } + +// if(!seawater.room){ +// continue; +// } + if(!seawater.room){ + seawater.room = null + } + + if(seawater.room){ + if(seawater.room>=11 || seawater.room<1){ + continue; + } + } + + if(!seawater.hall){ + seawater.hall = null + } + if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){ + continue; + } + if(!seawater.toilet){ + seawater.toilet = null + } + if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){ + continue; + } + + + def floor = parseFloor(seawater.floor_info) + if(seawater.height && floor && floor.isNumber()){ + if(floor.toInteger()>seawater.height){ + floor = null + seawater.height = null + } + + } + + if(!floor){ + seawater.floor = null + } + if(floor && floor.toInteger()<1){ + floor = null + } + + def rowMap = [:]; + if(cat == "rent"){ + rowMap.rent_type = seawater.rent_type + if(seawater.avg_price && seawater.avg_price>1000){ + continue + } + + if(seawater.avg_price && seawater.avg_price<10){ + continue + } + + } + rowMap.houseId = seawater.id + rowMap.city = CcbCommonUtils.getCcbCity(city) + //rowMap.region =seawater.region + //rowMap.address = seawater.address + + + rowMap.area = seawater.area + rowMap.total_price = seawater.total_price + rowMap.avg_price = seawater.avg_price + rowMap.height = seawater.height=="null"?null:seawater.height + if(rowMap.height && rowMap.height<1){ + rowMap.height = null + } + rowMap.floor = floor=="null"?null:floor + rowMap.room = seawater.room=="null"?null:seawater.room + rowMap.hall = seawater.hall=="null"?null:seawater.hall + rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet + rowMap.room_structure = seawater.room_structure=="null"?null:seawater.room_structure + if(CcbCommonUtils.rootMap.get(seawater.root)){ + rowMap.root = CcbCommonUtils.rootMap.get(seawater.root) + }else{ + rowMap.root = "99" + } + + def direction = parseDirection(seawater.direction) + rowMap.direction = direction=="null"?null:direction + + def fitment = parseFitment(seawater.fitment) + rowMap.fitment = fitment=="null"?null:fitment + + if(seawater.p_date instanceof String){ + seawater.p_date = seawater.p_date.toInteger() + } + if(seawater.p_date<=20200101){ + //seawater.p_date.toString().replace("2019","2020")// + rowMap.p_date = DateUtils.getRandomDate("2020-01-01", "2020-07-31") + rowMap.c_date = rowMap.p_date + rowMap.m_date = rowMap.p_date + }else{ + rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") + rowMap.c_date = DateUtils.getConverDate(seawater.m_date) + rowMap.m_date = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-18 10:00:00"//DateUtils.getConverDate(seawater.m_date) + } + + rets << rowMap + +// if(houseIds.contains(seawater.id)){ +// println seawater.id +" 已存在" +// }else{ +// rets << rowMap +// } + +// def count = isExistFromHouse(seawater.id) +// if(count){ +// println seawater.id +" 已存在" +// }else{ +// rets << rowMap +// } + + } + } + + println name +" "+houses.size()+" last:"+rets.size() + return rets; + } + + + def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){ + def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd") + def startMdate = DateUtils.parseFromTightByPattern(date,"yyyyMMdd") + def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd") + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(date.toInteger()).to(endDate).includeLower(false).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); + + + QueryBuilder date_range_mdate = QueryBuilders.rangeQuery("m_date").from(startMdate.getTime()).to(endMdate.getTime()).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range).must(date_range_mdate)//.must(QueryBuilders.termQuery("root", "lianjia.com")) + + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + + + if(name.contains("弄小区")){ + def newName = name.replace("弄小区","弄") + if(!alias.contains(newName)){ + alias << newName + } + } + + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"] + def index = new Random().nextInt(6); + + def rets = [] + def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder); + + if(houses){ + + def ids = new HashSet() + for(def h : rets1){ + ids << h.houseId + } + for (int k=0;k60){ + continue; + } + +// if(!seawater.room){ +// continue; +// } + if(!seawater.room){ + seawater.room = null + } + + if(seawater.room){ + if(seawater.room>=11 || seawater.room<1){ + continue; + } + } + + if(!seawater.hall){ + seawater.hall = null + } + if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){ + continue; + } + if(!seawater.toilet){ + seawater.toilet = null + } + if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){ + continue; + } + + + def floor = parseFloor(seawater.floor_info) + if(seawater.height && floor && floor.isNumber()){ + if(floor.toInteger()>seawater.height){ + floor = null + seawater.height = null + } + + } + + if(!floor){ + seawater.floor = null + } + if(floor && floor.toInteger()<1){ + floor = null + } + + def rowMap = [:]; + if(cat == "rent"){ + rowMap.rent_type = seawater.rent_type + if(seawater.avg_price && seawater.avg_price>1000){ + continue + } + + if(seawater.avg_price && seawater.avg_price<10){ + continue + } + + } + rowMap.houseId = seawater.id + rowMap.city = CcbCommonUtils.getCcbCity(city) + //rowMap.region =seawater.region + //rowMap.address = seawater.address + + + rowMap.area = seawater.area + rowMap.total_price = seawater.total_price + rowMap.avg_price = seawater.avg_price + rowMap.height = seawater.height=="null"?null:seawater.height + if(rowMap.height && rowMap.height<1){ + rowMap.height = null + } + rowMap.floor = floor=="null"?null:floor + rowMap.room = seawater.room=="null"?null:seawater.room + rowMap.hall = seawater.hall=="null"?null:seawater.hall + rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet + rowMap.room_structure = seawater.room_structure=="null"?null:seawater.room_structure + if(CcbCommonUtils.rootMap.get(seawater.root)){ + rowMap.root = CcbCommonUtils.rootMap.get(seawater.root) + }else{ + rowMap.root = "99" + } + + def direction = parseDirection(seawater.direction) + rowMap.direction = direction=="null"?null:direction + + def fitment = parseFitment(seawater.fitment) + rowMap.fitment = fitment=="null"?null:fitment + + if(seawater.p_date instanceof String){ + seawater.p_date = seawater.p_date.toInteger() + } + if(seawater.p_date<=20200101){ + //seawater.p_date.toString().replace("2019","2020")// + rowMap.p_date = DateUtils.getRandomDate("2020-01-01", "2020-07-31") + rowMap.c_date = rowMap.p_date + rowMap.m_date = rowMap.p_date + }else{ + rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") + rowMap.c_date = DateUtils.getConverDate(seawater.m_date) + rowMap.m_date = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-18 10:00:00"//DateUtils.getConverDate(seawater.m_date) + } + + rets << rowMap + +// if(houseIds.contains(seawater.id)){ +// println seawater.id +" 已存在" +// }else{ +// rets << rowMap +// } + +// def count = isExistFromHouse(seawater.id) +// if(count){ +// println seawater.id +" 已存在" +// }else{ +// rets << rowMap +// } + } + } + + println name +" "+houses.size()+" last:"+rets.size() + return rets; + } + + /** + * 是否是别墅 + * @return + */ + public boolean isVilla(area, name, height, room) { + if (area == null) { + return false; + } + if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) { + return true; + } + + boolean ret = false; + if (height != null && room != null) { + if (height <= 3) { + if (area > 200 && room >= 5) { + ret = true; + } + } + } else if (height == null && room != null) { + if (area > 360 && room >= 6) { + ret = true; + } + } else if (height == null && room == null) { + if (area > 500) { + ret = true; + } + } + + return ret; + } + + def getPublicDate(def pDate){ + + if(pDate>=20181001 && pDate<20200201){ + return DateUtils.getRandomDate("2018-10-01", "2020-02-01") + } + def pdate = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(pDate), "yyyy-MM-dd HH:mm:ss") + return pdate; + } + + static def parseFloor(def floorDesc){ + if(floorDesc){ + floorDesc = floorDesc.replaceAll("[\\s ]+", "") + if(floorDesc){ + Pattern pat1 = Pattern.compile("第(\\d{1,2})层"); + Matcher matcher1 = pat1.matcher(floorDesc); + if (matcher1.find()) { + return matcher1.group(1) + } + pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?"); + matcher1 = pat1.matcher(floorDesc); + if (matcher1.find()) { + return matcher1.group(1) + } +// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){ +// return "中层" +// } +// +// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){ +// return "高层" +// } +// +// +// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){ +// return "低层" +// } + if(floorDesc.isNumber()){ + return floorDesc; + } + } + } + return null; + } + + private String csvHandlerStr(String str) { + //csv格式如果有逗号,整体用双引号括起来;如果里面还有双引号就替换成两个双引号,这样导出来的格式就不会有问题了 + String tempDescription=str; + //如果有逗号 + if(str.contains(",")||str.contains(",")){ + //如果还有双引号,先将双引号转义,避免两边加了双引号后转义错误 + if(str.contains("\"")){ +// tempDescription=str.replace("\"", "\"\""); + tempDescription=str.replace("\"", ""); + } + //在将逗号转义 + tempDescription="\""+tempDescription+"\""; + } + return tempDescription; + } + + + static String parseFitment(String fitmentDesc) { + if (fitmentDesc == null) { + return null + } + fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "") + if (fitmentDesc.equals("非毛坯")) { + return null + } + Iterator>> it = FITMENTS_TYPES_MAPS + .entrySet().iterator() + Map.Entry> entry + while (it.hasNext()) { + entry = it.next() + if (fitmentDesc.contains(entry.getKey())) { + return entry.getKey() + } + for (String fitmentKey : entry.getValue()) { + if (fitmentDesc.contains(fitmentKey)) { + return entry.getKey() + } + } + } + return null + + } + + static String parseDirection(String directionDesc) { + if (directionDesc == null) { + return null + } + directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#") + StringBuffer sb = new StringBuffer() + for (int i = 0; i < directionDesc.length(); i++) { + String c = directionDesc.substring(i, i + 1) + if (DIRECTIONS.contains(c)) { + if (directionDesc.substring(i).startsWith(c + "#_#")) { + sb.append(c + " ") + } else { + sb.append(c) + } + } + } + for (String d : DIRECTIONS) { + if (sb.toString().startsWith(d)) { + return d + } + } + for (String d : DIRECTIONS) { + if (sb.toString().contains(d)) { + return d + } + } + return null + + } + + + + + def marge(){ + File file = new File("D:\\数据\\建行\\新建文本文档.txt"); + def ids = new HashSet(); + def index = 0 + + file.eachLine{it-> + index++ + if(index>1){ + def arrs = it.split("\\|@\\|") + def id = arrs[1] + if(!ids.contains(id)){ + ids << id + } + } + } + println index + println ids.size() + } + + def getAllHouseIds(def commid, def cityCode){ + def sql = "SELECT FORSALEID FROM bigdata_forsale_gj WHERE cityid='"+cityCode+"' and commid = '"+commid+"'"; + + def houseIds = new HashSet() + sqlInstance.eachRow(sql,{it-> + houseIds << it.FORSALEID + }); + return houseIds; + } + + def findAllHouseIdsByCity(){ + // 创建布隆过滤器对象 + BloomFilter filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001); +// def houseIds = new HashSet() + pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.t0831_bigdata_forsale_gj_h",{it-> + filter.put(it.gujia_sell_id) + }); + + pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.t0831_bigdata_forsale_gj_h_2021",{it-> + filter.put(it.gujia_sell_id) + }); + return filter; + } + + + def isExistFromHouse(def houseId){ + def sql = "SELECT count(FORSALEID) as count FROM bigdata_forsale_gj WHERE FORSALEID = '"+houseId+"'"; + + def count = null + sqlInstance.eachRow(sql,{it-> + count = it.count + }); + return count; + } + + + def saveToPgSql(rows){ + for(def r : rows) { + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ + r[i] = null + } + } + r << null + r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28" + r << null + r << '2999-12-31' + r << null + r << null + r << null + } + + println "start: "+new Date() + def sql = "insert into base.bigdata_forsale values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + pgsqlInstance.withBatch(20000, sql){ ps-> + for(def r : rows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], + r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39], r[40], + r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49], r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58]) + ps.toString() + } + } + + println "end: "+new Date() + } + +} From 06c1f7c1d779c132d3767c037904517482128a73 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 15 May 2024 18:16:43 +0800 Subject: [PATCH 3/5] =?UTF-8?q?=E5=8A=A0=E5=9F=8E=E5=B8=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 3409b89b4..cee43a5b9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -16,7 +16,7 @@ import groovy.sql.Sql class CcbCommonUtils { // - public static def citys_fjw_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山"] + public static def citys_fjw_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山","阿拉尔"] public static def citys_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"] public static def citys_92 = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"] public static def citys_252 = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"] From fda9911b3a7264f9bbef2fd4aa1fd21271e564f3 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 16 May 2024 13:22:24 +0800 Subject: [PATCH 4/5] =?UTF-8?q?=E6=88=BF=E6=BA=90=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/CcbCommonUtils.groovy | 4 +- .../ccbevaluation/PgsqlCommUtils.groovy | 6 + .../FjwBigdataCommbuildtypeBatch.groovy | 5 +- .../FjwBigdataForsaleBatch.groovy | 253 ++---------------- .../fjwPriceBaseData/FjwPgsqlCommUtils.groovy | 25 ++ 5 files changed, 56 insertions(+), 237 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwPgsqlCommUtils.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index cee43a5b9..3e18b6b7f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -16,7 +16,7 @@ import groovy.sql.Sql class CcbCommonUtils { // - public static def citys_fjw_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山","阿拉尔"] + public static def citys_fjw_all = [ "乌鲁木齐","上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山","阿拉尔"] public static def citys_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"] public static def citys_92 = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"] public static def citys_252 = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"] @@ -439,7 +439,7 @@ class CcbCommonUtils { q.put("cat", cat); q.put("city", city); // q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13 -// q.put("name","汤堡小区") +// q.put("name","新泾三村") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy index cfbd78a2d..0a1584be9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy @@ -2,6 +2,9 @@ package com.fangjia.script.ccbevaluation import groovy.sql.Sql +/** + * 建行使用 + */ class PgsqlCommUtils { public static def database = "warehouse" @@ -19,6 +22,9 @@ class PgsqlCommUtils { } + + + public static def findAllDistrictCount(def cityid){ def sql = "select count(gujia_comm_id) from base.t0831_bigdata_community_gj_h where urbn_id='${cityid}' and gujia_bds_bds_st=0"; def count = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommbuildtypeBatch.groovy index b76f4139a..16e99a043 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommbuildtypeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommbuildtypeBatch.groovy @@ -3,9 +3,10 @@ package com.fangjia.script.gj.fjwPriceBaseData import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher +import com.fangjia.script.ccbevaluation.CcbCommonUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.ccb.CcbCodeUtils -import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.gj.fjwPriceBaseData.FjwPgsqlCommUtils import com.fangjia.script.util.DateUtils import groovy.sql.Sql @@ -38,7 +39,7 @@ class FjwBigdataCommbuildtypeBatch extends BaseBatch{ def cityCodeDatas = CcbCommonUtils.findCityCodeValue() for(def city : citys){ - def distIds = PgsqlCommUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city)) + def distIds = FjwPgsqlCommUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city)) def rows = createDatas(cityCodeDatas.get(city), distIds) rets.addAll(rows) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy index 908bdc375..2b8cde742 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy @@ -64,10 +64,10 @@ class FjwBigdataForsaleBatch extends BaseBatch{ public static ExecutorService fixedThreadPool2 = Executors.newFixedThreadPool(200) - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", - "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") +// def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", +// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") - def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/fangjia?stringtype=unspecified", + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified", "postgres", "Goojia123456", "org.postgresql.Driver") @@ -204,64 +204,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{ } - def repeatHouseId(allDatas){ - def totalCount = new AtomicLong() - final CountDownLatch latch = new CountDownLatch(allDatas.size()) - def rets = new CopyOnWriteArrayList() - for(def i=0;i60){ - continue; - } - -// if(!seawater.room){ -// continue; -// } - if(!seawater.room){ - seawater.room = null - } - - if(seawater.room){ - if(seawater.room>=11 || seawater.room<1){ - continue; - } - } - - if(!seawater.hall){ - seawater.hall = null - } - if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){ - continue; - } - if(!seawater.toilet){ - seawater.toilet = null - } - if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){ - continue; - } - - - def floor = parseFloor(seawater.floor_info) - if(seawater.height && floor && floor.isNumber()){ - if(floor.toInteger()>seawater.height){ - floor = null - seawater.height = null - } - - } - - if(!floor){ - seawater.floor = null - } - if(floor && floor.toInteger()<1){ - floor = null - } - def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room) - if(vallaFlag){ - continue; - } - - def rowMap = [:]; - if(cat == "rent"){ - rowMap.rent_type = seawater.rent_type - if(seawater.avg_price && seawater.avg_price>1000){ - continue - } - - if(seawater.avg_price && seawater.avg_price<10){ - continue - } - - } - rowMap.houseId = seawater.id - rowMap.city = city - //rowMap.region =seawater.region - //rowMap.address = seawater.address - - - rowMap.area = seawater.area - rowMap.total_price = seawater.total_price - rowMap.avg_price = seawater.avg_price - rowMap.height = seawater.height=="null"?null:seawater.height - if(rowMap.height && rowMap.height<1){ - rowMap.height = null - } - rowMap.floor = floor=="null"?null:floor - rowMap.room = seawater.room=="null"?null:seawater.room - rowMap.hall = seawater.hall=="null"?null:seawater.hall - rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet - if(CcbCommonUtils.rootMap.get(seawater.root)){ - rowMap.root = CcbCommonUtils.rootMap.get(seawater.root) - }else{ - rowMap.root = "99" - } - - def direction = parseDirection(seawater.direction) - rowMap.direction = direction=="null"?null:direction - - def fitment = parseFitment(seawater.fitment) - rowMap.fitment = fitment=="null"?null:fitment - - - - if(seawater.p_date instanceof String){ - seawater.p_date = seawater.p_date.toInteger() - } - if(seawater.p_date<=20200101){ - //seawater.p_date.toString().replace("2019","2020")// - rowMap.p_date = DateUtils.getRandomDate("2020-01-01", "2020-07-31") - rowMap.c_date = rowMap.p_date - rowMap.m_date = rowMap.p_date - }else{ - rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") - rowMap.c_date = DateUtils.getConverDate(seawater.m_date) - rowMap.m_date = DateUtils.getConverDate(seawater.m_date) - } - - rets << rowMap - } - } - - println commId +" "+houses.size()+" last:"+rets.size() - return rets; - } def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, def commId, def cityCodeDataMap){ @@ -700,9 +487,9 @@ class FjwBigdataForsaleBatch extends BaseBatch{ def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){ - def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd") - def startMdate = DateUtils.parseFromTightByPattern(date,"yyyyMMdd") - def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd") + def date = "20200101"//DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd") + def startMdate = DateUtils.parseFromTightByPattern("20200101","yyyyMMdd")//DateUtils.parseFromTightByPattern(date,"yyyyMMdd") + def endMdate = DateUtils.parseFromTightByPattern("20240518", "yyyyMMdd")//DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd") QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(date.toInteger()).to(endDate).includeLower(false).includeUpper(false); QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); @@ -710,9 +497,19 @@ class FjwBigdataForsaleBatch extends BaseBatch{ QueryBuilder date_range_mdate = QueryBuilders.rangeQuery("m_date").from(startMdate.getTime()).to(endMdate.getTime()).includeLower(true).includeUpper(false); + //用以下方法会查出发布时间很久以前的 +// QueryBuilder dateQueryBuilder = QueryBuilders.boolQuery() +// dateQueryBuilder.should(date_range) +// dateQueryBuilder.should(date_range_mdate) +// QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) +// .must(area_range).must(dateQueryBuilder) + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) .must(date_range).must(area_range).must(date_range_mdate)//.must(QueryBuilders.termQuery("root", "lianjia.com")) + + if(!alias){ alias = [] } @@ -1094,27 +891,17 @@ class FjwBigdataForsaleBatch extends BaseBatch{ // 创建布隆过滤器对象 BloomFilter filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001); // def houseIds = new HashSet() - pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.t0831_bigdata_forsale_gj_h",{it-> + pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.bigdata_forsale",{it-> filter.put(it.gujia_sell_id) }); - pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.t0831_bigdata_forsale_gj_h_2021",{it-> - filter.put(it.gujia_sell_id) - }); +// pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.bigdata_forsale_2021",{it-> +// filter.put(it.gujia_sell_id) +// }); return filter; } - def isExistFromHouse(def houseId){ - def sql = "SELECT count(FORSALEID) as count FROM bigdata_forsale_gj WHERE FORSALEID = '"+houseId+"'"; - - def count = null - sqlInstance.eachRow(sql,{it-> - count = it.count - }); - return count; - } - def saveToPgSql(rows){ for(def r : rows) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwPgsqlCommUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwPgsqlCommUtils.groovy new file mode 100644 index 000000000..10c94f0ea --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwPgsqlCommUtils.groovy @@ -0,0 +1,25 @@ +package com.fangjia.script.gj.fjwPriceBaseData + +import groovy.sql.Sql + +/** + * 房价网内部使用 + */ +class FjwPgsqlCommUtils { + + public static def database = "fangjia" + + public static def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/"+database+"?stringtype=unspecified", + "postgres", "Goojia123456", "org.postgresql.Driver") + + public static def findAllDistrictIdsByOracle(def cityid){ + def sql = "select gujia_comm_id from base.bigdata_community where urbn_id='${cityid}' and gujia_bds_bds_st=0"; + def set = new HashSet() + sqlInstance.eachRow(sql,{r-> + set << r.gujia_comm_id + }); + return set + } + + +} From a63ea19635f618f793ca96e9074625c2bb3aa549 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 16 May 2024 14:26:08 +0800 Subject: [PATCH 5/5] =?UTF-8?q?=E6=97=A7=E6=88=90=E4=BA=A4=E5=8C=B9?= =?UTF-8?q?=E9=85=8D=EF=BC=8C=E5=89=8D60=E4=B8=AA=E6=9C=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/BigdataDealBatch.groovy | 2 +- .../script/gj/data_handle/DevProcess.groovy | 15 +- .../FjwBigdataDealBatch.groovy | 819 ++++++++++++++++++ 3 files changed, 834 insertions(+), 2 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataDealBatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index a45e49793..9b22ad4b9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -34,7 +34,7 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataDealBatch extends BaseBatch{ - def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -24, "yyyyMM")); + def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -60, "yyyyMM")); def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index 342e946fb..f24f72702 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -22,6 +22,9 @@ class DevProcess { DBCollection DEV_MAPPING_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL) + + DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com") + static main(args) { def task = new DevProcess(); // task.run() @@ -38,7 +41,8 @@ class DevProcess { // task.stat() // task.addNewDev() // task.createDevMapping() - task.deleteAddr() +// task.deleteAddr() + task.synchDevTo43() } def parseCityAndDist(){ @@ -482,4 +486,13 @@ class DevProcess { DEV_MAPPING_CONN.remove(new BasicDBObject("_id", new ObjectId(d['A']))) }); } + + def synchDevTo43(){ + def dbcursor = development_zone_CONN.find() + def rets = [] + for(def r : dbcursor){ + rets << r + } + DEVELOPMENT_ZONE_43_COLL.insert(rets) + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataDealBatch.groovy new file mode 100644 index 000000000..ebb2ccf1f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataDealBatch.groovy @@ -0,0 +1,819 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.core.RegionData +import com.fangjia.core.util.DataUtil +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.rabbitmq.client.impl.AMQImpl +import groovy.sql.Sql + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Pattern + +/** + * 建行模型估值项目 + * 成交数据同步 + */ +@BatchOwner(Watcher.GaoJuan) +class FjwBigdataDealBatch extends BaseBatch{ + + def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -24, "yyyyMM")); + def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + + def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com", "幸福里":"xflapp.com"] + + public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; + public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl) + + DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") + + static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified", + "postgres", "postgres", "org.postgresql.Driver") + + static main(args) { + runBatch(new FjwBigdataDealBatch(), args); + } + + @Override + public void process(Object args) { + + def citys = CcbCommonUtils.getRunningCitys(args) + + println "curDate:"+curDate + def allCitys = CommonDatautils.findAllCity() + for(def city : allCitys){ + addDealPriceMatch(city); + } + +// def rows = [] +// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] +// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +// def index = 0 +// for(def city : citys){ +// index ++ +// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) +// def ppeDists = [:] +// +// def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city)) +// println "历史id加载量:"+dealIds.size() +// def statMap = [:]//getDistrictStatByCity(city, weekend) +// if(city=="上海"){ +// def youDataTrade = getYouDaData(statMap) +// def youdaDatas = createData(city, youDataTrade, cityCodeDataMap, distCodeMap, "youda", dealIds) +// rows.addAll(youdaDatas) +// println city +" "+youdaDatas.size() +// } +// +// def trades = getCityData(city, statMap, ppeDists) +// if(trades){ +// def rets = createData(city, trades, cityCodeDataMap, distCodeMap, "other", dealIds) +// rows.addAll(rets) +// println "【${index}】"+city +" 成交共:"+rets.size() +// +// def cityDatas = [] +// cityDatas << headers +// cityDatas.addAll(rows) +// } +// +// } +// +// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" +// if(CommonDatautils.isWindow()){ +// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" +// } +// +// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); +// println "txt 生成 " + rows.size() +// +//// delPgData() +// saveToPgSql(rows) +// +//// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat") +//// saveToPgSql2(datas) + } + + def readTxt(def fileName){ + File file = new File(fileName); + def rows = []; + + file.eachLine{ + def row = it.split("\\|@\\|") + rows << row + } + return rows; + } + + + def createData(city, def rets, cityCodeDatas, distCodeMap, source, dealIds){ + def result = Collections.synchronizedList(new ArrayList()); + dealIds = Collections.synchronizedList(dealIds); + + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(rets.size()) + +// def lianjiaPriceMap = findThirdPartyPrice("上海") +// def startDate = DateUtils.parseFromTightByPattern(DateUtils.addMonth(new Date(),-2),'yyyy-MM-dd') +// def lianjiaPriceMap = getDistrictStatByCity("上海", startDate) + + def ccbPriceMap = null + if(city == "上海"){ + ccbPriceMap = getCommunityPrices("上海") + } + + for(def i=0;i 3000) { + price = db.lianjia_esf_price + } + if (!price && db.beike_esf_price && db.beike_esf_price > 3000) { + price = db.beike_esf_price + } + if(price){ + map.put(db.region+"##"+db.name, price) + } + + } + return map + } + + def converYouDaData(def trade, def cityCodeDatas, def distCodeMap, def ccbPriceMap){ + def ret = [] + def obj = CcbCommonUtils.findDistrict(trade.fjCity, trade.fjRegion, trade.fjName, "district") + if(!obj){ + return ret; + } + def price = ccbPriceMap.get(obj._id.toString()) + if(price){ + def mod = java.lang.Math.abs((trade.price-price)*100/price) + if(mod<=20){ + ret << cityCodeDatas.get(trade.city) + ret << trade._id.toString() + ret << trade.city + def data = distCodeMap.get(trade.fjRegion) + if(data){ + ret << data.district_code + ret << data.district + }else{ + ret << "" + ret << "" + } + def street = "" + def streetNumber = "" + if(trade.address){ + if(trade.address.toString().contains("弄")){ + String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄")) + if(number){ + street = trade.address.toString().substring(0, trade.address.toString().indexOf(number)) + streetNumber = number + } + } + if(!street){ + def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name) + if(jsonObj && jsonObj.searchAddress){ + if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.contains("弄")){ + String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄")) + if(number){ + street = trade.address.toString().substring(0, trade.address.toString().indexOf(number)) + streetNumber = number + } + }else{ + street = jsonObj.searchAddress.street + streetNumber = jsonObj.searchAddress.streetNumber + } + + } + } + } + + if(!street){ + if(obj.address){ + if(obj.address.toString().contains("弄")){ + String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄")) + if(number){ + street = obj.address.toString().substring(0, obj.address.toString().indexOf(number)) + streetNumber = number + } + }else{ + def jsonObj = OpenApiClient.addressMatch(obj.city, obj.address, "property", false, trade.region, obj.name) + if(jsonObj && jsonObj.searchAddress){ + street = jsonObj.searchAddress.street + streetNumber = jsonObj.searchAddress.streetNumber + } + } + } + } + + ret << street + ret << streetNumber + + + if(obj){ + ret << obj._id.toString() + ret << trade.fjName + } + + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << CcbCodeUtils.parseHouseProperty(obj.house_roperty)//权属 + ret << CcbCodeUtils.parseHousePropertyYear(obj.property_life) + ret << ""//期数 + String roomNum = DataUtil.match(trade.address, Pattern.compile("\\d+室")) + if(roomNum && roomNum.contains("0")){ + ret << roomNum.substring(0, roomNum.lastIndexOf("0")) + }else{ + ret << trade.floor + } + + ret << "" + ret << trade.height + def arrs = CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型 + def buildType = '' + if(arrs){ + buildType = arrs[0] + } + ret << buildType + ret << CcbCodeUtils.getBuildStructureCode(obj.build_structure) + ret << trade.area + ret << "" + ret << getHouseRoom(buildType, trade.room, trade.hall) + ret << trade.room + ret << trade.hall + ret << "" + ret << trade.toilet + ret << CcbCodeUtils.parseHouseUse(obj.estate_type2) + ret << "" + ret << CcbCodeUtils.getDirectionCode(trade.direction) + ret << CcbCodeUtils.getFitmentCode(trade.fitment) + ret << "" + ret << trade.price + ret << trade.totalPrice * 10000 + ret << "" + ret << "" + ret << "" + ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd") + ret << "02" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "99" + + ret << "" + ret << "" + ret << "" + ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss") + ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + ret << trade.totalPrice * 10000 + ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd") + } + } + + return ret; + } + + + def converData(def trade, def cityCodeDatas, def distCodeMap){ + def ret = [] + def data = distCodeMap.get(trade.region) + if(!data){ + println trade.city +" "+trade.region+" 没有找到对应的区域" + return ret + } + ret << cityCodeDatas.get(trade.city) + ret << trade._id.toString() + ret << CcbCommonUtils.getCcbCity(trade.city) + if(data){ + ret << data.district_code + ret << data.district + }else{ + ret << "" + ret << "" + } + + if(trade.address){ + if(trade.address.toString().contains("弄")){ + String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄")) + if(number){ + ret << trade.address.toString().substring(0, trade.address.toString().indexOf(number)) + ret << number + }else{ + ret << "" + ret << "" + } + }else{ + def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name) + if(jsonObj && jsonObj.searchAddress){ + ret << jsonObj.searchAddress.street + ret << jsonObj.searchAddress.streetNumber + }else{ + ret << "" + ret << "" + } + } + }else{ + ret << "" + ret << "" + } + ret << trade.dist_id.toString() + ret << trade.master_name + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << CcbCodeUtils.parseHouseProperty(trade.house_roperty)//权属 + ret << CcbCodeUtils.parseHousePropertyYear(trade.property_life) + ret << ""//期数 + ret << trade.floor + ret << "" + ret << trade.height +// ret << CcbCodeUtils.parseBuildType(trade.build_type)//建筑类型 + def arrs = CcbCodeUtils.parseBuildType(trade.build_type)//建筑类型 + def buildType = '' + if(arrs){ + buildType = arrs[0] + } + ret << buildType + ret << CcbCodeUtils.getBuildStructureCode(trade.build_structure) + ret << trade.area + ret << "" + ret << getHouseRoom(buildType, trade.room, trade.hall) + ret << trade.room + ret << trade.hall + ret << "" + ret << trade.toilet + ret << CcbCodeUtils.parseHouseUse(trade.estate_type2) + ret << "" + ret << CcbCodeUtils.getDirectionCode(trade.direction) + ret << CcbCodeUtils.getFitmentCode(trade.fitment) + ret << "" + ret << trade.avg_price + ret << trade.total_price + ret << "" + ret << "" + ret << "" + ret << DateUtils.formatDateByPattern(trade.trade_date, "yyyy-MM-dd") + ret << "02" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + def root = sourceMap.get(trade.source) + if(root){ + if(CcbCommonUtils.rootMap.get(root)){ + ret << CcbCommonUtils.rootMap.get(root) + }else{ + ret << "99" + } + }else{ + ret << "" + } + + ret << "" + ret << "" + ret << "" + ret << DateUtils.formatDateByPattern(trade.create_date, "yyyy-MM-dd HH:mm:ss") + ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + ret << trade.total_price + ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd") + return ret; + } + + def getHouseRoom(buildtype, def roomcnt, def livingroomcnt){ + def housetype = null + if(buildtype && ["0112","0110","0109"].contains(buildtype)){ + housetype = "07" + }else if(buildtype && ["0107","0106","0105"].contains(buildtype)){ + housetype = "16" + }else{ + if(roomcnt && roomcnt==4){ + if(livingroomcnt>1){ + housetype = "14" + }else{ + housetype = "11" + } + }else if(roomcnt && roomcnt==3){ + if(livingroomcnt>1){ + housetype = "13" + }else{ + housetype = "10" + } + }else if(roomcnt && roomcnt==2){ + if(livingroomcnt>1){ + housetype = "12" + }else{ + housetype = "09" + } + }else if(roomcnt && roomcnt==1){ + housetype = "08" + }else{ + housetype="99" + } + } + return housetype; + } + + def getCityData(def city, def statMap, def ppeDistsMap){ + + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"], null) + println city +" 小区量:"+dists.size() + def tradesList = Collections.synchronizedList(new ArrayList()); + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(dists.size()) + +// def citys = ["北京","广州","深圳","重庆","郑州","西安","成都","东莞","佛山","杭州","合肥","南京","长春","扬州","无锡","太原","台州","泉州","宁波","南通","南昌","贵阳","哈尔滨","惠州","济南","金华","昆明","遵义","镇江","肇庆","宜昌","芜湖","泰州","汕头","清远","秦皇岛","马鞍山","洛阳","连云港","九江","淮安","呼和浩特","沧州","保定","安庆"] + + def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) + + def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date()) + for(def i=0;i=startPrice && obj.price<=endPrice){ +// rets << obj; +// }else{ +// index++ +// println index + " "+obj +// } +// } + } + } + return rets; + } + + + def getDistrictStatByCity(city, weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("weekend", new BasicDBObject('$gte', weekend)) + def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("weekend",-1)) + def map = new ConcurrentHashMap() + for(def d : dbcursor){ + def price = map.get(d.region+"##"+d.district_name) + if(!price && d.value_type==2){ + map.put(d.region+"##"+d.district_name, d.avg_price) + } + } + return map + } + + def findHistoryDealIds(def city){ + def sql = "SELECT gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where urbn_nm='${city}'" + def ids = [] + try{ + pgsqlInstance.eachRow(sql,{ + ids << it.gujia_bds_mdl_id + }); + }catch(Exception e){ + + } + + return ids + } + + def getPpeGTE15Datas(def city){ + def sql = "select unit_price,commid from avm_evaluation_item_gj where data_type='成交' and pe>=15 and pe<=20 " + + "and cityname='"+city+"'"; + def map = [:] + try{ + sqlInstance.eachRow(sql,{ + map.put(it.commid, it.unit_price) + }); + }catch(Exception e){ + getPpeGTE15Datas(city) + } + + return map + } + + + def addDealPriceMatch(city){ +// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) + def startDate = DateUtils.parseFromTightByPattern(startDate.toString()+"01", "yyyyMMdd") + def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) +// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210101", "yyyyMMdd")) +// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20221001", "yyyyMMdd")) + def trades = CcbCommonUtils.findTradesByMonth(city, startDate) + + final CountDownLatch latch = new CountDownLatch(trades.size()) + def final AtomicInteger count_ = new AtomicInteger(0) + + for(def i=0;i + for(def r : rows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], + r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39], + r[40], r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49],r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58], r[59], + r[60], r[61], r[62], r[63], r[64], r[65], r[66], r[67], r[68], r[69],r[70], r[71]) + } + } + + println "end: "+new Date() + } + + + def saveToPgSql2(rows){ + def newRows = [] + for(def r : rows) { + def newR = [] + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ +// r[i] = null + newR << null + }else{ + newR << r[i] + } + } + newR << null + newR << "2023-09-28" + newR << null + newR << '2999-12-31' + newR << null + newR << null + newR << null + + newRows << newR + } + + println "start: "+new Date() + def sql = "insert into base.t0831_bigdata_deal_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + pgsqlInstance.withBatch(20000, sql){ ps-> + for(def r : newRows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], + r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39], + r[40], r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49],r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58], r[59], + r[60], r[61], r[62], r[63], r[64], r[65], r[66], r[67], r[68], r[69],r[70], r[71]) + } + } + + println "end: "+new Date() + } +}