From 74c1bb4b998f3032ee1b995c9987e4152f692ead Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 25 Oct 2022 13:44:12 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=E5=AF=BC=E5=87=BA=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/ExportOracleData.groovy | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index e1e17ee47..462d76787 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -25,7 +25,7 @@ class ExportOracleData { def run(args) { citys = CcbCommonUtils.getRunningCitys(args) -// def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); + def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") def new_dists = [] @@ -41,7 +41,7 @@ class ExportOracleData { exportBigdataCommunity()// exportCommunityStaging()//没cityid exportCommunityBuileType()// - +// exportThirdPartyPrice2(lastMonth, new_dists)// exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 @@ -51,23 +51,23 @@ class ExportOracleData { //计算结果相关 -// exportAvmEvaluationGJNB(curMonth) -// exportAVMEvaluationItemGJNB(curMonth) -// -// exportCityPrice(curMonth) -// exportRegionPrice(curMonth) -// -// exportCommunityBasePriceGj(curMonth)//已合下面的合并 -// exportCommunityBasePrice(curMonth) -// -// exportCommunityPdescribeGj(curMonth) -// exportCommCaseDetailGj(curMonth) + exportAvmEvaluationGJNB(curMonth) + exportAVMEvaluationItemGJNB(curMonth) -// def curDate = curMonth.toString()+"28" -// SSHHelper jschCommand = new SSHHelper(); -// System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}")); - //ok文件 -// exportLastFile() + exportCityPrice(curMonth) + exportRegionPrice(curMonth) + + exportCommunityBasePriceGj(curMonth)//已合下面的合并 + exportCommunityBasePrice(curMonth) + + exportCommunityPdescribeGj(curMonth) + exportCommCaseDetailGj(curMonth) + + def curDate = curMonth.toString()+"28" + SSHHelper jschCommand = new SSHHelper(); + System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}")); +// ok文件 + exportLastFile() } @@ -376,7 +376,7 @@ class ExportOracleData { "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + "from bigdata_deal_gj where CITYID='${cityCode}' and dealdate>=to_date('${dealDate}','yyyy-MM-dd')"; - allRows = dealData(sql) + allRows.addAll(dealData(sql)) } } From eaf35c26ac74cc43bff7a8a3ba5be88e6f6c6517 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 25 Oct 2022 13:44:46 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E7=AC=AC=E4=B8=89=E6=96=B9=E4=BF=AE?= =?UTF-8?q?=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../thirdPriceCrawl/DataStandardBatch.groovy | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index 91d3dee7e..7c1ad2c31 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -91,11 +91,9 @@ class DataStandardBatch extends BaseBatch{ // ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl) cityMap.each{city,alias-> -// if(city == "北京"){ - println "开始加载【${city}】" - def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"]) - districtMap.putAll(detailMap) -// } + println "开始加载【${city}】" + def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"]) + districtMap.putAll(detailMap) } while(flag){ @@ -122,9 +120,9 @@ class DataStandardBatch extends BaseBatch{ def final AtomicInteger count_ = new AtomicInteger(0) def final AtomicInteger successIndex = new AtomicInteger(0) datas.each { dbo -> - fixedThreadPool.execute( - new Runnable() { - void run() { +// fixedThreadPool.execute( +// new Runnable() { +// void run() { try{ println "(${rowIndex}-${count}-${count_.incrementAndGet()})【${dbo.city}-${dbo.region}-${dbo.name}-${dbo.address}-${dbo.source}】" if(dbo.name){ @@ -164,12 +162,12 @@ class DataStandardBatch extends BaseBatch{ }finally { latch.countDown() } - } - } - ); +// } +// } +// ); } - latch.await() +// latch.await() long end = System.currentTimeMillis() println "第【${rowIndex}】次中匹配成功的:"+successCount+" 个,耗时:"+ (end - start)+" "+ DateUtils.format(new Date(),"yyyy-MM-dd HH:mm:ss") @@ -502,6 +500,8 @@ class DataStandardBatch extends BaseBatch{ type = 12;//区域不同 district = null; }else{ + def score = CosineSimilarAlgorithm.getSimilarity(district.name, copyObj.name) + district.score = score //此处用来做测试 type=15 } } @@ -922,7 +922,7 @@ class DataStandardBatch extends BaseBatch{ qdata.put("s_date", SHORT_DATE) // qdata.put("city","北京") // qdata.put("source","安居客") -// qdata.put("name",Pattern.compile("金隅澜湾.*")) +// qdata.put("name",Pattern.compile("龚家宅.*")) qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false)); def ret = [] try{ @@ -1065,6 +1065,9 @@ class DataStandardBatch extends BaseBatch{ udata2.put("match_region", district.region); udata2.put("match_name", district.name); udata2.put("match_address", district.address); + if(district.score){ + udata2.put("score", district.score); + } udata2.put("avg_price", data.avg_price); udata2.put("type", type); udata2.put("tid", data._id.toString()); From 17f7722530be18c2f63f70d36a893ea28dc95010 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 25 Oct 2022 13:45:44 +0800 Subject: [PATCH 3/3] update --- .../ccbevaluation/BigdataDealBatch.groovy | 2 +- .../ccbevaluation/GuiDingPriceBatch.groovy | 4 +-- .../fangjia/script/ccbevaluation/Test.groovy | 28 +++++++++---------- .../script/gj/chongqingblank/DataMatch.groovy | 25 ++++++++++++++++- .../fangjia/script/util/OpenApiClient.groovy | 16 +++++++++++ 5 files changed, 57 insertions(+), 18 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 89b9f7f35..82972cdd3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -58,7 +58,7 @@ class BigdataDealBatch extends BaseBatch{ public void process(Object args) { //def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date()) // def citys = CcbCommonUtils.citys; - def citys = CcbCommonUtils.getRunningCitys(args) + def citys = ["上海"]//CcbCommonUtils.getRunningCitys(args) if(args && args.size()>=2){ curDate = args[1].toString().toInteger() } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 936022f98..c390e6c83 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -36,7 +36,7 @@ class GuiDingPriceBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + def citys = CcbCommonUtils.getRunningCitys(args) def rows = [] for (def city : citys) { @@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2022-09-22", "2022-09-23") + def prices = CcbCommonUtils.findPersonPrice(city, "2022-10-21", "2022-10-22") if(prices){ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 31ae30039..e987ea5af 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -1001,10 +1001,10 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business -//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\录入人工查价_0922_300.xlsx" +def fileName = "D:\\数据\\建行\\房产估值\\指导价\\人工指导价录入_1020.xlsx" -//ExcelUtils.multiThreadingModify(fileName, {d-> -// if(d['A']){ +ExcelUtils.multiThreadingModify(fileName, {d-> + if(d['A']){ // def region = RegionData.formatRegion(d['A'], d['B']) // if(region){ // def dist = CommonDatautils.findDistrict(d['A'], region, d['C'], "district") @@ -1013,18 +1013,18 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business // d['F'] = dist._id.toString() // } // } -//// if(!d['Q']){ -//// def dist = CommonDatautils.findDistrictById(d['A']) -////// -//// if(dist){ -//// d['H'] = dist.region -////// d['G'] = dist.name -//// d['I'] = dist.status -//// } -//// } + if(!d['F']){ + def dist = CommonDatautils.findDistrictById(d['A']) // -// } -//},0) + if(dist){ +// d['H'] = dist.region +// d['G'] = dist.name + d['F'] = dist.status + } + } + + } +},0) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy index 418b6c102..bba57a816 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy @@ -29,7 +29,30 @@ class DataMatch { // task.run3() // println task.findAddress("昌州街道棠香路") - task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx") +// task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx") + task.fzWeiyuanshuMatch("D:\\数据\\福州\\商品住房楼盘评估指导参考价情况表_匹配.xlsx") + } + + def fzWeiyuanshuMatch(fileName){ + ExcelUtils.modify(fileName,{d-> + if(!d['K'] && d['C']){ + def arr = OpenApiClient.search("福州", d['G'], d['C']) + if(arr){ + d['H'] = arr.city + d['I'] = arr.district + d['J'] = arr.name + } + } + +// def results = null +// if(!d['K'] && d['C']){ +// results = OpenApiClient.addressMatch("福州",d['C'], "property", false, "长乐", null) +// } +// if(results && results.precise>=0){ +// d['L'] = results.district +// d['M'] = results.name +// } + },0) } def run1(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index 68cd266c8..cc7b1ad50 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -52,6 +52,22 @@ public class OpenApiClient { return false } + public static def search(def city, def district, def name){ + Map params = new HashMap() + params.put("city", city); + params.put("keyword", name); + params.put("district", district) + params.put("token", TOKEN); + String content = new HttpRequest().get("http://open.fangjia.com/property/search", params, "utf-8") + JSONObject jsonObj = JSONObject.fromObject(content) + JSONArray array = jsonObj.get("result") + def matchObj = null + if(array){ + matchObj = array[0] + } + return matchObj + } + public static List searchDistrict(double lng, double lat, int r, String city) { DecimalFormat format = new DecimalFormat("#.0000000000"); try {