From 74c1bb4b998f3032ee1b995c9987e4152f692ead Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 25 Oct 2022 13:44:12 +0800 Subject: [PATCH 01/15] =?UTF-8?q?=E5=AF=BC=E5=87=BA=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/ExportOracleData.groovy | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index e1e17ee47..462d76787 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -25,7 +25,7 @@ class ExportOracleData { def run(args) { citys = CcbCommonUtils.getRunningCitys(args) -// def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); + def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") def new_dists = [] @@ -41,7 +41,7 @@ class ExportOracleData { exportBigdataCommunity()// exportCommunityStaging()//没cityid exportCommunityBuileType()// - +// exportThirdPartyPrice2(lastMonth, new_dists)// exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 @@ -51,23 +51,23 @@ class ExportOracleData { //计算结果相关 -// exportAvmEvaluationGJNB(curMonth) -// exportAVMEvaluationItemGJNB(curMonth) -// -// exportCityPrice(curMonth) -// exportRegionPrice(curMonth) -// -// exportCommunityBasePriceGj(curMonth)//已合下面的合并 -// exportCommunityBasePrice(curMonth) -// -// exportCommunityPdescribeGj(curMonth) -// exportCommCaseDetailGj(curMonth) + exportAvmEvaluationGJNB(curMonth) + exportAVMEvaluationItemGJNB(curMonth) -// def curDate = curMonth.toString()+"28" -// SSHHelper jschCommand = new SSHHelper(); -// System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}")); - //ok文件 -// exportLastFile() + exportCityPrice(curMonth) + exportRegionPrice(curMonth) + + exportCommunityBasePriceGj(curMonth)//已合下面的合并 + exportCommunityBasePrice(curMonth) + + exportCommunityPdescribeGj(curMonth) + exportCommCaseDetailGj(curMonth) + + def curDate = curMonth.toString()+"28" + SSHHelper jschCommand = new SSHHelper(); + System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}")); +// ok文件 + exportLastFile() } @@ -376,7 +376,7 @@ class ExportOracleData { "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + "from bigdata_deal_gj where CITYID='${cityCode}' and dealdate>=to_date('${dealDate}','yyyy-MM-dd')"; - allRows = dealData(sql) + allRows.addAll(dealData(sql)) } } From eaf35c26ac74cc43bff7a8a3ba5be88e6f6c6517 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 25 Oct 2022 13:44:46 +0800 Subject: [PATCH 02/15] =?UTF-8?q?=E7=AC=AC=E4=B8=89=E6=96=B9=E4=BF=AE?= =?UTF-8?q?=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../thirdPriceCrawl/DataStandardBatch.groovy | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index 91d3dee7e..7c1ad2c31 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -91,11 +91,9 @@ class DataStandardBatch extends BaseBatch{ // ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl) cityMap.each{city,alias-> -// if(city == "北京"){ - println "开始加载【${city}】" - def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"]) - districtMap.putAll(detailMap) -// } + println "开始加载【${city}】" + def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"]) + districtMap.putAll(detailMap) } while(flag){ @@ -122,9 +120,9 @@ class DataStandardBatch extends BaseBatch{ def final AtomicInteger count_ = new AtomicInteger(0) def final AtomicInteger successIndex = new AtomicInteger(0) datas.each { dbo -> - fixedThreadPool.execute( - new Runnable() { - void run() { +// fixedThreadPool.execute( +// new Runnable() { +// void run() { try{ println "(${rowIndex}-${count}-${count_.incrementAndGet()})【${dbo.city}-${dbo.region}-${dbo.name}-${dbo.address}-${dbo.source}】" if(dbo.name){ @@ -164,12 +162,12 @@ class DataStandardBatch extends BaseBatch{ }finally { latch.countDown() } - } - } - ); +// } +// } +// ); } - latch.await() +// latch.await() long end = System.currentTimeMillis() println "第【${rowIndex}】次中匹配成功的:"+successCount+" 个,耗时:"+ (end - start)+" "+ DateUtils.format(new Date(),"yyyy-MM-dd HH:mm:ss") @@ -502,6 +500,8 @@ class DataStandardBatch extends BaseBatch{ type = 12;//区域不同 district = null; }else{ + def score = CosineSimilarAlgorithm.getSimilarity(district.name, copyObj.name) + district.score = score //此处用来做测试 type=15 } } @@ -922,7 +922,7 @@ class DataStandardBatch extends BaseBatch{ qdata.put("s_date", SHORT_DATE) // qdata.put("city","北京") // qdata.put("source","安居客") -// qdata.put("name",Pattern.compile("金隅澜湾.*")) +// qdata.put("name",Pattern.compile("龚家宅.*")) qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false)); def ret = [] try{ @@ -1065,6 +1065,9 @@ class DataStandardBatch extends BaseBatch{ udata2.put("match_region", district.region); udata2.put("match_name", district.name); udata2.put("match_address", district.address); + if(district.score){ + udata2.put("score", district.score); + } udata2.put("avg_price", data.avg_price); udata2.put("type", type); udata2.put("tid", data._id.toString()); From 17f7722530be18c2f63f70d36a893ea28dc95010 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 25 Oct 2022 13:45:44 +0800 Subject: [PATCH 03/15] update --- .../ccbevaluation/BigdataDealBatch.groovy | 2 +- .../ccbevaluation/GuiDingPriceBatch.groovy | 4 +-- .../fangjia/script/ccbevaluation/Test.groovy | 28 +++++++++---------- .../script/gj/chongqingblank/DataMatch.groovy | 25 ++++++++++++++++- .../fangjia/script/util/OpenApiClient.groovy | 16 +++++++++++ 5 files changed, 57 insertions(+), 18 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 89b9f7f35..82972cdd3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -58,7 +58,7 @@ class BigdataDealBatch extends BaseBatch{ public void process(Object args) { //def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date()) // def citys = CcbCommonUtils.citys; - def citys = CcbCommonUtils.getRunningCitys(args) + def citys = ["上海"]//CcbCommonUtils.getRunningCitys(args) if(args && args.size()>=2){ curDate = args[1].toString().toInteger() } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 936022f98..c390e6c83 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -36,7 +36,7 @@ class GuiDingPriceBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + def citys = CcbCommonUtils.getRunningCitys(args) def rows = [] for (def city : citys) { @@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2022-09-22", "2022-09-23") + def prices = CcbCommonUtils.findPersonPrice(city, "2022-10-21", "2022-10-22") if(prices){ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 31ae30039..e987ea5af 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -1001,10 +1001,10 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business -//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\录入人工查价_0922_300.xlsx" +def fileName = "D:\\数据\\建行\\房产估值\\指导价\\人工指导价录入_1020.xlsx" -//ExcelUtils.multiThreadingModify(fileName, {d-> -// if(d['A']){ +ExcelUtils.multiThreadingModify(fileName, {d-> + if(d['A']){ // def region = RegionData.formatRegion(d['A'], d['B']) // if(region){ // def dist = CommonDatautils.findDistrict(d['A'], region, d['C'], "district") @@ -1013,18 +1013,18 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business // d['F'] = dist._id.toString() // } // } -//// if(!d['Q']){ -//// def dist = CommonDatautils.findDistrictById(d['A']) -////// -//// if(dist){ -//// d['H'] = dist.region -////// d['G'] = dist.name -//// d['I'] = dist.status -//// } -//// } + if(!d['F']){ + def dist = CommonDatautils.findDistrictById(d['A']) // -// } -//},0) + if(dist){ +// d['H'] = dist.region +// d['G'] = dist.name + d['F'] = dist.status + } + } + + } +},0) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy index 418b6c102..bba57a816 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy @@ -29,7 +29,30 @@ class DataMatch { // task.run3() // println task.findAddress("昌州街道棠香路") - task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx") +// task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx") + task.fzWeiyuanshuMatch("D:\\数据\\福州\\商品住房楼盘评估指导参考价情况表_匹配.xlsx") + } + + def fzWeiyuanshuMatch(fileName){ + ExcelUtils.modify(fileName,{d-> + if(!d['K'] && d['C']){ + def arr = OpenApiClient.search("福州", d['G'], d['C']) + if(arr){ + d['H'] = arr.city + d['I'] = arr.district + d['J'] = arr.name + } + } + +// def results = null +// if(!d['K'] && d['C']){ +// results = OpenApiClient.addressMatch("福州",d['C'], "property", false, "长乐", null) +// } +// if(results && results.precise>=0){ +// d['L'] = results.district +// d['M'] = results.name +// } + },0) } def run1(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index 68cd266c8..cc7b1ad50 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -52,6 +52,22 @@ public class OpenApiClient { return false } + public static def search(def city, def district, def name){ + Map params = new HashMap() + params.put("city", city); + params.put("keyword", name); + params.put("district", district) + params.put("token", TOKEN); + String content = new HttpRequest().get("http://open.fangjia.com/property/search", params, "utf-8") + JSONObject jsonObj = JSONObject.fromObject(content) + JSONArray array = jsonObj.get("result") + def matchObj = null + if(array){ + matchObj = array[0] + } + return matchObj + } + public static List searchDistrict(double lng, double lat, int r, String city) { DecimalFormat format = new DecimalFormat("#.0000000000"); try { From b137e0f08fbc3edbff3eadf49b0fb7441ecea2f8 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 26 Oct 2022 10:59:26 +0800 Subject: [PATCH 04/15] =?UTF-8?q?=E5=8E=BB=E6=8E=89=E4=BA=86=E4=B8=80?= =?UTF-8?q?=E4=BA=9B=E7=8E=B0=E5=9C=A8=E4=B8=8D=E9=9C=80=E8=A6=81=E7=9A=84?= =?UTF-8?q?=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ThirdDataToGatherBatch.groovy | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy index 3efe71b67..850ef9d42 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy @@ -81,8 +81,8 @@ class ThirdDataToGatherBatch extends BaseBatch{ println "小区添加价格完成" districts = addOtherWebsitePrice(districts); println "小区添加第三方价格完成" - districts = addLevel(districts); - println "小区评级完成" +// districts = addLevel(districts); +// println "小区评级完成" saveOrUpdate(districts); println city + "保存修改完成" toLineDB(city) @@ -101,16 +101,16 @@ class ThirdDataToGatherBatch extends BaseBatch{ def tempPrice = d.get(field+"_price") if(!tempPrice){ d.put(field+"_price", m.avg_price.toInteger()) - if(d.fangjia_price && m.avg_price){ - if(d.fangjia_price){ - try{ - d.put(field+"_diff", diffPercent(d.fangjia_price.toInteger(), m.avg_price.toInteger())) - }catch(Exception e){ - println "dddd" - } - - } - } +// if(d.fangjia_price && m.avg_price){ +// if(d.fangjia_price){ +// try{ +// d.put(field+"_diff", diffPercent(d.fangjia_price.toInteger(), m.avg_price.toInteger())) +// }catch(Exception e){ +// println "dddd" +// } +// +// } +// } } }else{ @@ -208,8 +208,9 @@ class ThirdDataToGatherBatch extends BaseBatch{ query.put("s_date", SHORT_DATE) DBObject udata = new BasicDBObject() - udata.put("level",d.level); - udata.put("last_price", d.last_price); +// udata.put("level",d.level); +// udata.put("last_price", d.last_price); + udata.put("district_id", d._id.toString()) udata.put("fangjia_price", d.fangjia_price); fieldMap.each{k,v-> From da38bead89c598b7f7760707d1770ee17f862cd8 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 26 Oct 2022 17:35:13 +0800 Subject: [PATCH 05/15] =?UTF-8?q?=E5=BB=BA=E8=A1=8C=E7=9B=B8=E5=85=B3?= =?UTF-8?q?=E8=84=9A=E6=9C=AC=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../BigdataCommunityBatch.groovy | 43 +++++++------------ .../ccbevaluation/BigdataForsaleBatch.groovy | 1 - .../script/ccbevaluation/CcbFtpUtils.groovy | 8 ++++ .../ccbevaluation/ExportOracleData.groovy | 38 ++++++++-------- .../gj/district/FangjiaClientUtils.groovy | 17 ++++++++ .../fangjia/script/jiangnanblank/SFTP.groovy | 16 +++++++ .../ThirdDataToGatherBatch.groovy | 2 +- .../fangjia/script/util/OpenApiClient.groovy | 1 + 8 files changed, 77 insertions(+), 49 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index c06cef98e..a96bb9b09 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -8,6 +8,7 @@ import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.ccb.CcbCodeUtils import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.OpenApiClient @@ -175,38 +176,25 @@ class BigdataCommunityBatch extends BaseBatch{ ret << "pa101" ret << "" if(dist.address){ - if(dist.address.toString().contains("弄")){ - String number = DataUtil.match(dist.address, Pattern.compile("\\d+弄")) - if(number){ - ret << dist.address.toString().substring(0, dist.address.toString().indexOf(number)) - ret << dist.address.toString().substring(dist.address.toString().indexOf(number), dist.address.toString().size()) + def jsonObj = null + if(dist.address){ + jsonObj = FangjiaClientUtils.addressStructure(dist.city, dist.address) + } + if(jsonObj && jsonObj.street){ + if(jsonObj.street){ + ret << jsonObj.street }else{ ret << "" + } + + if(jsonObj.streetNumber){ + ret << jsonObj.streetNumber + }else{ ret << "" } }else{ - def jsonObj = null - if(dist.address){ -// jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name) - jsonObj = OpenApiClient.standard1Action(dist.city, dist.address) - } - if(jsonObj && jsonObj.street){ - if(jsonObj.street){ - ret << jsonObj.street - }else{ - ret << "" - } - - if(jsonObj.streetNumber){ - ret << jsonObj.streetNumber - }else{ - ret << "" - } - - }else{ - ret << "" - ret << "" - } + ret << "" + ret << "" } }else{ ret << "" @@ -225,7 +213,6 @@ class BigdataCommunityBatch extends BaseBatch{ println dist.city + " "+dist.name+" error" e.printStackTrace() } - return ret; } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index ca517f31a..5b0edbae1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -448,7 +448,6 @@ class BigdataForsaleBatch extends BaseBatch{ if(!alias.contains(newName)){ alias << newName } - } if(alias && alias.size()>0){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy index 6b972553e..124911f34 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy @@ -3,6 +3,7 @@ package com.fangjia.script.ccbevaluation import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.jiangnanblank.SFTP import com.jcraft.jsch.ChannelSftp +import com.jcraft.jsch.SftpATTRS import org.apache.commons.codec.digest.DigestUtils class CcbFtpUtils { @@ -64,6 +65,13 @@ class CcbFtpUtils { return files } + static def getFileLength(String path, String month){ + sftpChannel = reConnect(); + def size = sftp.getFileSize(ftpPath+ month+"/"+path, sftpChannel) + return size + } + + static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index eabf59e2a..22861ba1d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -30,24 +30,24 @@ class ExportOracleData { def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") def new_dists = [] //不会变 10.25 -// exportEvaluationRateData() -// exportBigdataPoiData()// -// exportBigdataDistrictData()// -// -// exportGuidedPrice()//随人工改变// -// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策// -// -//// //处理小区情况 -// exportBigdataCommunity()// -// exportCommunityStaging()//没cityid + exportEvaluationRateData() + exportBigdataPoiData()// + exportBigdataDistrictData()// + + exportGuidedPrice()//随人工改变// + exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策// + +// //处理小区情况 + exportBigdataCommunity()// + exportCommunityStaging()//没cityid exportCommunityBuileType()// -// -// exportThirdPartyPrice2(lastMonth, new_dists)// -// exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 -// exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 -// -// //第三方网站权重表 -// exportWebsiteRate(); + + exportThirdPartyPrice2(lastMonth, new_dists)// + exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 + exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 + + //第三方网站权重表 + exportWebsiteRate(); // // // //计算结果相关 @@ -229,8 +229,8 @@ class ExportOracleData { FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "BIGDATA_COMMBUILDTYPE_GJ 数据文件输出完成......" -// CcbFtpUtils.ftpUpload(null, fileName, curDate) -// println "BIGDATA_COMMBUILDTYPE_GJ 上传Ftp完成......" + CcbFtpUtils.ftpUpload(null, fileName, curDate) + println "BIGDATA_COMMBUILDTYPE_GJ 上传Ftp完成......" } def exportBigdataDistrictData() { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 9f3670a44..ae1dee2e4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -67,6 +67,7 @@ class FangjiaClientUtils { + //分词 省,市,区 public static def addressStand(def city, def address){ def jsonObj = null def content = null @@ -84,6 +85,22 @@ class FangjiaClientUtils { return jsonObj; } + //分词基础上的标准化,complete=false 不不全 short=true 省市区返回简拼 + public static def addressStructure(def city, def address){ + def jsonObj = null + def content = null + try { + address = address.replace("{","") + def url = "http://192.168.0.19:31080/FangJiaAddressService/structure/v2?address="+URLEncoder.encode(address,"UTF-8")+"&city="+URLEncoder.encode(city,"UTF-8") + content = new HttpRequest().newGet(url, null, "utf-8") + jsonObj = JSONObject.fromObject(content) + } catch (Exception e) { + println address +" "+content + e.printStackTrace(); + } + return jsonObj; + } + public static def arroundPrice(city,address){ def jsonObj = null def content = null diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy index 70082d882..a50827e27 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy @@ -209,4 +209,20 @@ class SFTP { return isDirExistFlag; } + public long getFileSize(String srcSftpFilePath, ChannelSftp sftp) { + long fileSize;//文件大于等于0则存在 + try { + SftpATTRS sftpATTRS = sftp.lstat(srcSftpFilePath); + fileSize = sftpATTRS.getSize(); + } catch (Exception e) { + e.printStackTrace() + fileSize = -1;//获取文件大小异常 + if (e.getMessage().toLowerCase().equals("no such file")) { + fileSize = -2;//文件不存在 + } + } + return fileSize; + } + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy index 850ef9d42..320e65b2f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy @@ -210,7 +210,7 @@ class ThirdDataToGatherBatch extends BaseBatch{ DBObject udata = new BasicDBObject() // udata.put("level",d.level); // udata.put("last_price", d.last_price); - udata.put("district_id", d._id.toString()) + udata.put("seaweed_id", d._id.toString()) udata.put("fangjia_price", d.fangjia_price); fieldMap.each{k,v-> diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index cc7b1ad50..7c37cae97 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -364,6 +364,7 @@ public class OpenApiClient { } + //此接口地址标准化,带补全 public static JSONObject standard1Action(String city, String address) { try { Map params = new HashMap(); From 5873c62bab26abcc9c744955d22b6c6e65000be7 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 27 Oct 2022 14:17:13 +0800 Subject: [PATCH 06/15] update --- .../BigdataCommunityBatch.groovy | 13 ++++--- .../ccbevaluation/CcbFtpTestUtils.groovy | 16 +++++++++ .../ExportOracleEvaluationData.groovy | 9 ++--- .../ccbevaluation/GuiDingPriceBatch.groovy | 2 +- .../script/gj/chongqingblank/DataMatch.groovy | 36 +++++++++---------- 5 files changed, 46 insertions(+), 30 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index a96bb9b09..fafe61ba9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -36,7 +36,7 @@ class BigdataCommunityBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.citys_92; + def citys = CcbCommonUtils.getRunningCitys(args) def rows = [] @@ -108,7 +108,6 @@ class BigdataCommunityBatch extends BaseBatch{ if(ret){ rets << ret } - } catch (Exception e) { e.printStackTrace() } finally { @@ -180,15 +179,15 @@ class BigdataCommunityBatch extends BaseBatch{ if(dist.address){ jsonObj = FangjiaClientUtils.addressStructure(dist.city, dist.address) } - if(jsonObj && jsonObj.street){ - if(jsonObj.street){ - ret << jsonObj.street + if(jsonObj && jsonObj.result && jsonObj.result.street){ + if(jsonObj.result.street){ + ret << jsonObj.result.street }else{ ret << "" } - if(jsonObj.streetNumber){ - ret << jsonObj.streetNumber + if(jsonObj.result.streetNumber){ + ret << jsonObj.result.streetNumber }else{ ret << "" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy index 7083cb615..b9f4572e5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy @@ -70,6 +70,22 @@ class CcbFtpTestUtils { } + static def listFileNames(month){ + //上传到ftp + sftpChannel = reConnect(); + Vector files = sftp.listFiles(ftpPath+ month+"/", sftpChannel); + return files + } + + static def getFileLength(String path, String month){ + sftpChannel = reConnect(); + def size = sftp.getFileSize(ftpPath+ month+"/"+path, sftpChannel) + return size + } + + + + static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy index 7258934fa..b37ee5b94 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy @@ -12,13 +12,14 @@ class ExportOracleEvaluationData { def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + def citys = null static main(args) { def task = new ExportOracleEvaluationData() - task.run() + task.run(args) } - def run() { - + def run(args) { + citys = CcbCommonUtils.getRunningCitys(args) def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); //计算结果相关 @@ -369,7 +370,7 @@ class ExportOracleEvaluationData { def exportCommCaseDetailGj(month){ - def citys = CcbCommonUtils.citys; +// def citys = CcbCommonUtils.citys; def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index c390e6c83..6981c7149 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2022-10-21", "2022-10-22") + def prices = CcbCommonUtils.findPersonPrice(city, "2022-10-22", "2022-10-27") if(prices){ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy index bba57a816..cb615604a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy @@ -30,28 +30,28 @@ class DataMatch { // println task.findAddress("昌州街道棠香路") // task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx") - task.fzWeiyuanshuMatch("D:\\数据\\福州\\商品住房楼盘评估指导参考价情况表_匹配.xlsx") + task.fzWeiyuanshuMatch("D:\\数据\\福州\\连江楼盘.xlsx") } def fzWeiyuanshuMatch(fileName){ - ExcelUtils.modify(fileName,{d-> - if(!d['K'] && d['C']){ - def arr = OpenApiClient.search("福州", d['G'], d['C']) - if(arr){ - d['H'] = arr.city - d['I'] = arr.district - d['J'] = arr.name - } - } + ExcelUtils.multiThreadingModify(fileName,{d-> +// if(!d['G']){ +// def arr = OpenApiClient.search("福州", "连江", d['C']) +// if(arr){ +// d['E'] = arr.city +// d['F'] = arr.district +// d['G'] = arr.name +// } +// } -// def results = null -// if(!d['K'] && d['C']){ -// results = OpenApiClient.addressMatch("福州",d['C'], "property", false, "长乐", null) -// } -// if(results && results.precise>=0){ -// d['L'] = results.district -// d['M'] = results.name -// } + def results = null + if(d['H'] && d['H']=="F" && d['C']){ + results = OpenApiClient.addressMatch("福州",d['C'], "property", false, "连江", null) + } + if(results && results.precise>=0){ + d['J'] = results.district + d['K'] = results.name + } },0) } From aa25d429d2db8797ad0ea6ca7f6572ad847501c5 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 27 Oct 2022 14:21:21 +0800 Subject: [PATCH 07/15] delete --- .../src/main/java/com/fangjia/core/util/DataUtil.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java b/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java index adfb2c7c8..4aaabb88f 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java @@ -25,8 +25,6 @@ import java.util.zip.GZIPOutputStream; import com.hankcs.hanlp.HanLP; import com.hankcs.hanlp.dictionary.py.Pinyin; -import com.sun.javafx.scene.transform.TransformUtils; -import net.sf.json.JSON; import net.sf.json.JSONArray; import net.sf.json.JSONNull; import net.sf.json.JSONObject; From 22c2852dcd1d53e2911ede14ef5b11c6a5b19693 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 27 Oct 2022 16:37:02 +0800 Subject: [PATCH 08/15] =?UTF-8?q?=E6=89=B9=E9=87=8F=E5=9C=B0=E5=9D=80?= =?UTF-8?q?=E6=8B=86=E5=88=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/data/AddressCut.groovy | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy new file mode 100644 index 000000000..e34c2b4af --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy @@ -0,0 +1,80 @@ +package com.fangjia.script.ccbevaluation.data + +import com.fangjia.script.BaseBatch +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.DbUtils +import com.mongodb.BasicDBObject + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class AddressCut extends BaseBatch{ + static main(args) { + runBatch(new AddressCut(), args); + } + + @Override + public void process(Object args) { + def citys = CommonDatautils.findAllCity() + def index = 0 + + def fileName = "/home/goojia/gj/address_split.txt" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\address_split.txt" + } + for(def c : citys) { + index++ + def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city", "region", "name","address"]) + def rets = cut(dists) + FileUtils.batchCreateFile(rets, fileName, null, "utf-8", true); + } + } + + def cut(rets){ + ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + final CountDownLatch latch = new CountDownLatch(rets.size()) + def totalCount = new AtomicLong() + def rows = new CopyOnWriteArrayList() + for(def i=0;i Date: Thu, 27 Oct 2022 16:44:03 +0800 Subject: [PATCH 09/15] add logs --- .../com/fangjia/script/ccbevaluation/data/AddressCut.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy index e34c2b4af..e836327b2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy @@ -33,6 +33,7 @@ class AddressCut extends BaseBatch{ def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city", "region", "name","address"]) def rets = cut(dists) FileUtils.batchCreateFile(rets, fileName, null, "utf-8", true); + println index +" "+ c +" "+ rets.size() } } From 661871bee9f57fb577692cb2bc979760d24d342d Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 27 Oct 2022 16:47:28 +0800 Subject: [PATCH 10/15] add logs --- .../com/fangjia/script/ccbevaluation/data/AddressCut.groovy | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy index e836327b2..f8770132c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy @@ -61,12 +61,16 @@ class AddressCut extends BaseBatch{ street_number = jsonObj.result.streetNumber } if(street || street_number){ + + println totalCount.get() +" "+d.city +" "+d.address +" "+street+" "+street_number + rows << [d.city,d.region, d.name, d.address, street, street_number] // udata.put("m_date", new Date()) // DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) } } } + } catch (Exception e) { e.printStackTrace() } finally { From 61b7ac8fca3518e96936bd88d24dcdf70f04ec9d Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 28 Oct 2022 15:58:05 +0800 Subject: [PATCH 11/15] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=BB=BA=E8=A1=8C86?= =?UTF-8?q?=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/ExportOracleDataTest.groovy | 1171 +++++++++++++++++ .../ccbevaluation/data/AddressCut.groovy | 61 + .../ccbevaluation/ronghe/SSHHelperTest.groovy | 112 ++ 3 files changed, 1344 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/SSHHelperTest.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy new file mode 100644 index 000000000..a93ceb174 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy @@ -0,0 +1,1171 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.core.RegionData +import com.fangjia.script.ccbevaluation.ronghe.SSHHelper +import com.fangjia.script.ccbevaluation.ronghe.SSHHelperTest +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.jcraft.jsch.ChannelSftp +import groovy.sql.Sql +import org.apache.commons.codec.digest.DigestUtils + +import java.sql.ResultSet + +class ExportOracleDataTest { + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + def citys = null + + static main(args) { + def task = new ExportOracleDataTest() + task.run(args) + } + + def run(args) { + citys = CcbCommonUtils.getRunningCitys(args) + def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); + def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") + def new_dists = [] + //不会变 10.25 + exportEvaluationRateData() + exportBigdataPoiData()// + exportBigdataDistrictData()// + + exportGuidedPrice()//随人工改变// + exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策// + + //处理小区情况 + exportBigdataCommunity()// + exportCommunityStaging()//没cityid + exportCommunityBuileType()// + + exportThirdPartyPrice2(lastMonth, new_dists)// + exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 + exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 + + //第三方网站权重表 + exportWebsiteRate(); + + //计算结果相关 + exportAvmEvaluationGJNB(curMonth)//无城市区分 + exportAVMEvaluationItemGJNB(curMonth)//无城市区分 + + exportCityPrice(curMonth)//无城市区分 + exportRegionPrice(curMonth)//无城市区分 + + exportCommunityBasePriceGj(curMonth)//已合下面的合并 //无城市区分 + exportCommunityBasePrice(curMonth)//无城市区分 + + exportCommunityPdescribeGj(curMonth)//无城市区分 + exportCommCaseDetailGj(curMonth)//无城市区分 + + def curDate = curMonth.toString()+"28" + // ok文件 +// exportInOk() + SSHHelperTest jschCommand = new SSHHelperTest(); + System.out.println(jschCommand.executeCommand("sh /home/ccb/output/eval_package.sh ${curDate}")); + + } + + + def exportEvaluationRateData() { + def sql = "select ID,RATE_TYPE,RATE,RANK,CITY,REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from evaluate_rate_gj"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.RATE_TYPE + data << r.RATE + data << r.RANK + data << r.CITY + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["系数ID","系数类型","系数","排序","城市","备注","创建时间","更新时间","来源公司"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "上传Ftp完成......" + } + + + def exportWebsiteRate(){ + def sql = "select DATA_SOURCE_ID,DATA_SOURCE,DATA_SOURCE_MAP,WEIGHT,TYPE,SOURCECODE,CITYID,CITYNAME,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from WEBSITE_CONFIG_GJ"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.DATA_SOURCE_ID + data << r.DATA_SOURCE + data << r.DATA_SOURCE_MAP + data << r.WEIGHT + data << r.TYPE + data << r.SOURCECODE + data << r.CITYID + data << r.CITYNAME + data << r.CREATETIME + data << r.UPDATETIME + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["系数ID","系数类型","系数","排序","城市","备注","创建时间","更新时间","来源公司"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "上传Ftp完成......" + } + + + def exportBigdataPoiData() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID, DISTID, ID, POI_NAME, TYPE, TYPE_CODE, LATITUDE, LONGITUDE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from bigdata_poi_gj where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.CITYID + data << r.DISTID + data << r.ID + data << r.POI_NAME + data << r.TYPE + data << r.TYPE_CODE + data << r.LATITUDE + data << r.LONGITUDE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","来源公司"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "BIGDATA_POI_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "BIGDATA_POI_GJ 上传Ftp完成......" + } + + def exportCommunityStaging() { + + def allRows = [] + def sql = "select ID,COMMID,MASTER_COMMID,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_staging_gj"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMID + data << r.MASTER_COMMID + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["分期ID","小区ID","小区主期ID","备注","创建时间","更新时间","来源公司"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "COMMUNITY_STAGING_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "COMMUNITY_STAGING_GJ 上传Ftp完成......" + } + + def exportCommunityBuileType() { +// def citys = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"] +// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID, COMMID, BUILDTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, WORKDATE from bigdata_commbuildtype_gj where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.BUILDTYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.WORKDATE + allRows << data + }); + } + + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "BIGDATA_COMMBUILDTYPE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "BIGDATA_COMMBUILDTYPE_GJ 上传Ftp完成......" + } + + def exportBigdataDistrictData() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select DISTID, DISTNAME, DISTSPELL, CITYID, DISTCODE, SEQ, DISTTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE, LJXZQH, LJXZQHDM, XZQHDM, WORKDATE from bigdata_district_gj where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.DISTID + data << r.DISTNAME + data << r.DISTSPELL + data << r.CITYID + data << r.DISTCODE + data << r.SEQ + data << r.DISTTYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.LJXZQH + data << r.LJXZQHDM + data << r.XZQHDM + data << r.WORKDATE + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" + } + def headers = [] +//["行政区ID","行政区名称","行政区名称拼音","城市ID","国标码","显示顺序","行政区类别","备注","创建时间","更新时间","来源公司","逻辑行政区","逻辑行政区划代码","行政区划代码","数据时间"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "BIGDATA_DISTRICT_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "BIGDATA_DISTRICT_GJ 上传Ftp完成......" + } + + + def exportGuidedPrice() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE,AVG_PRICE, GUIDING_PERSON, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE from guided_price_gj where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMID + data << r.COMMNAME + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.DEALDATE + data << r.AVG_PRICE + data << r.GUIDING_PERSON + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" + } + def headers = [] +//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "GUIDED_PRICE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "GUIDED_PRICE_GJ 上传Ftp完成......" + } + + def exportCommunityGuidedPrice(def date) { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where CITYID='${cityCode}' and createtime>=to_date('${date}','yyyyMMdd')"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMID + data << r.COMMNAME + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.AVGPRICE + data << r.SOURCETYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导价格","类型","备注","创建时间","更新时间","来源公司"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "COMMUNITY_GUIDE_PRICE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "COMMUNITY_GUIDE_PRICE_GJ 上传Ftp完成......" + } + + def exportBigdataDeal(def dealDate, def new_dists) { + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + + "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + + "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + + "from bigdata_deal_gj where CITYID='${cityCode}' and dealdate>=to_date('${dealDate}','yyyy-MM-dd') and distid not in ${newstr}" + def rets1 = dealData(sql1) + + def sql2 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + + "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + + "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + + "from bigdata_deal_gj where CITYID='${cityCode}' and distid in ${newstr}" + def rets2 = dealData(sql2) + allRows.addAll(rets1) + allRows.addAll(rets2) + }else{ + def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + + "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + + "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + + "from bigdata_deal_gj where CITYID='${cityCode}' and dealdate>=to_date('${dealDate}','yyyy-MM-dd')"; + allRows.addAll(dealData(sql)) + } + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "BIGDATA_DEAL_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "BIGDATA_DEAL_GJ 上传Ftp完成......" + } + + + def dealData(def sql){ + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.CITYID + data << r.DEALID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.STREET + data << r.STREETNO + data << r.COMMID + data << r.COMMNAME + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.UNITID + data << r.UNITNO + data << r.UNITNAME + data << r.HOUSEID + data << r.ADDRESS + data << r.BUILDYEAR + data << r.PROPTYPE + data << r.PROPYEARS + data << r.PERIODNUM + data << r.FLOORNUM + data << r.FLOORTYPE + data << r.FLOORCNT + data << r.BUILDTYPE + data << r.BUILDSTRU + data << r.BUILDAREA + data << r.LIVINGAREA + data << r.HOUSESTYLE + data << r.ROOMCNT + data << r.LIVINGROOMCNT + data << r.KITCHENCNT + data << r.BATHROOMCNT + data << r.HOUSEUSE + data << r.HOUSESTRU + data << r.HOUSEASPECT + data << r.FITMENT + data << r.EVALUATENO + data << r.DEALUNITPRICE + data << r.DEALTOTALPRICE + data << r.DECLAREPRICE + data << r.EVALUATEPRICE + data << r.CONTRACTNO + data << r.DEALDATE + data << r.DEALTYPE + data << r.PAYTYPE + data << r.DEALPERIOD + data << r.IFELEVATOR + data << r.COMPAREAGARDEN + data << r.COMPAREATERRACE + data << r.COMPAREABALCONY + data << r.COMPAREABAYWINDOW + data << r.COMPAREAATTIC + data << r.COMPAREAGARAGE + data << r.COMPAREABASEMENT + data << r.COMPAREAOTHER + data << r.DATASOURCE + data << r.AGENCYID + data << r.AGENCYNAME + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.CJJE + data << r.WORKDATE + allRows << data + }); + return allRows + } + + def exportThirdPartyPrice() { + def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, ZHUGE_AVG_PRICE, ANJUKE_AVG_PRICE, FANG_AVG_PRICE, LIANJIA_AVG_PRICE, CITYHOUSE_AVG_PRICE, MAITIAN_AVG_PRICE, KE_AVG_PRICE, \n" + + "TAOFANG_AVG_PRICE, ZHONGYUAN_AVG_PRICE, TBS_AVG_PRICE, QFANG_AVG_PRICE, FANGTAN_AVG_PRICE, FANGDUODUO_AVG_PRICE, WIWJ_AVG_PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE\n" + + "from third_party_price_gj"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMID + data << r.COMMNAME + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.YEAR_MONTH + data << r.ZHUGE_AVG_PRICE + data << r.ANJUKE_AVG_PRICE + data << r.FANG_AVG_PRICE + data << r.LIANJIA_AVG_PRICE + data << r.CITYHOUSE_AVG_PRICE + data << r.MAITIAN_AVG_PRICE + data << r.KE_AVG_PRICE + data << r.TAOFANG_AVG_PRICE + data << r.ZHONGYUAN_AVG_PRICE + data << r.TBS_AVG_PRICE + data << r.QFANG_AVG_PRICE + data << r.FANGTAN_AVG_PRICE + data << r.FANGDUODUO_AVG_PRICE + data << r.WIWJ_AVG_PRICE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE.replaceAll(~'\\r', '') + allRows << data + }) + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] +//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "THIRD_PARTY_PRICE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "THIRD_PARTY_PRICE_GJ 上传数据文件完成......" + } + + + def exportThirdPartyPrice2(price_date, new_dists) { + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" +// fileName = "D:\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] + def index = 0 + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def allRows = [] + if(new_dists) { + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " + + "COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " + + "CITYID='${cityCode}' and price_date=${price_date} and distid not in ${newstr} "; + def rets1 = getThirdPartyDatas(sql1) + allRows.addAll(rets1) + + def sql2 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " + + "COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " + + "CITYID='${cityCode}' and distid in ${newstr} "; + def rets2 = getThirdPartyDatas(sql2) + allRows.addAll(rets2) + }else{ + def sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where CITYID='${cityCode}' and price_date=${price_date} "; + allRows = getThirdPartyDatas(sql) + } + if(allRows && allRows.size()>0){ + index = index + 1 + if(index==1){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + } + } + } + + println "BIGDATA_THIRD_PARTY_PRICE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + } + + def getThirdPartyDatas(def sql){ + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.CITYID + data << r.DISTID + data << r.COMMID + data << r.COMMNAME + data << r.DATA_SOURCE_ID + data << r.DATA_SOURCE + data << r.PRICE_DATE + data << r.COMMUNITY_PRICE + data << r.COMMUNITY_MAX_PRICE + data << r.COMMUNITY_MIN_PRICE + data << r.UPDATETIME + allRows << data + }) + return allRows; + } + + def exportBigdataCommunity(){ + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID, DISTID, COMMID, DISTNAME, COMMNAME, ADDRESS, LATITUDE, LONGITUDE, LANDAREA, BUILDAREA, GREENINGRATE, PLOTRATIO, PARKINGCNT, BUILDYEAR, HOUSEUSE, ZTS, WORKDATE, UPDATEFLAG, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE, \n" + + "\tCOMMALIAS, STREET, STREETNO, IFNEW, INTRODUCTION, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, SOURCECODE, XZJDB, LJXZQHDM, LJXZQH,STATUS from bigdata_community_gj where CITYID='${cityCode}' ";//where cityId in (350200,150100) + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.DISTID + data << r.COMMID + data << r.DISTNAME + data << r.COMMNAME + data << r.ADDRESS + data << r.LATITUDE + data << r.LONGITUDE + data << r.LANDAREA + data << r.BUILDAREA + data << r.GREENINGRATE + data << r.PLOTRATIO + data << r.PARKINGCNT + data << r.BUILDYEAR + data << r.HOUSEUSE + data << r.ZTS + data << r.WORKDATE + data << r.UPDATEFLAG + data << r.UPDATETIME + data << r.COMMTYPE + data << r.COMMALIAS + data << r.STREET + data << r.STREETNO + data << r.IFNEW + data << "" + data << r.REMARK + data << r.CREATETIME + data << r.SOURCECODE + data << r.XZJDB + data << r.LJXZQHDM + data << r.LJXZQH + data << r.STATUS + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途", + //"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司", + //"乡镇/街道办","逻辑行政区划代码","逻辑行政区划","状态"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "BIGDATA_COMMUNITY_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "BIGDATA_COMMUNITY_GJ 上传Ftp完成......" + } + + def exportBigdataForsale(publishDate, new_dists){ + //["北京","上海"] + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"] + def index = 0 + for(def city : citys){ + def cityCode = cityCodeMap.get(city) + + def allRows = [] + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj " + + "where cityId='${cityCode}' and publishtime>=to_date('${publishDate}','yyyy-MM-dd') and distid not in ${newstr}"; + def rets1 = getHouseDatas(sql1) + allRows.addAll(rets1) + }else{ + def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}' and publishtime>=to_date('${publishDate}','yyyy-MM-dd')"; + allRows = getHouseDatas(sql) + } + if(allRows.size()>0){ + index = index + 1 + if(index == 1){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + } + + } + } + + if(new_dists){//补充被修区域的数据 + def newstr = "('"+new_dists.join("','")+"')" + def sql2 = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH " + + "from bigdata_forsale_gj where distid in ${newstr}" + def rets2 = getHouseDatas(sql2) + if(rets2.size()>0){ + FileUtils.batchCreateFile(rets2, fileName, headers, "utf-8", true); + } + } + println "BIGDATA_FORSALE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "BIGDATA_FORSALE_GJ 上传Ftp完成......" + } + + def getHouseDatas(sql){ + def allRows = [] + def index = 0 + sqlInstance.eachRow(sql,{r-> + index = index+1 + println r.CITYNAME +" "+index + def data = [] + data << r.CITYID + data << r.FORSALEID + data << r.PROPCODE + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.UNITID + data << r.UNITNO + data << r.UNITNAME + data << r.HOUSEID + data << r.ADDRESS + data << r.BUILDYEAR + data << r.PROPTYPE + data << r.PROPYEARS + data << r.UNITPRICE + data << r.TOTALPRICE + data << r.PAYTYPE + data << r.FLOORNUM + data << r.FLOORCNT + data << r.BUILDTYPE + data << r.HOUSESTRU + data << r.BUILDSTRU + data << r.BUILDAREA + data << r.LIVINGAREA + data << r.HOUSESTYLE + data << r.ROOMCNT + data << r.LIVINGROOMCNT + data << r.KITCHENCNT + data << r.BATHROOMCNT + data << r.HOUSEASPECT + data << r.FITMENT + data << r.IFELEVATOR + data << r.ELEVATORHOUSE + data << r.PUBLISHTIME + data << r.EXPIRETIME + data << r.IFFIRST + data << r.DATASOURCE + data << r.AGENCYID + data << r.AGENCYNAME + data << r.PUBLISHER + data << r.PHONE + data << "" + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.ATTACH + allRows << data +// if(allRows.size()%200000==0){ +// FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); +// allRows.clear() +// } + }); + return allRows + } + + + def exportAvmEvaluationGJNB(yearMonth){ + def sql = "select ID,CITYID,CITYNAME, YEAR_MONTH, COMMUNITY_NUM, HAS_PRICE_COMMUNITY_NUM, PRICE_COVERAGE, ARTIFICIAL_CALC_NUM, ARTIFICIAL_CALC_COVERAGE, SAMPLE_CALC_NUM, SAMPLE_CALC_COVERAGE, \n" + + "ANN_CALC_NUM, ANN_CALC_COVERAGE, TEST_NUM, TEST_COMM_NUM, TEST_COMM_COVERAGE, PPE15_COVERAGE, RATE_MEDIAN, COEFFICIENT,DEAL_TEST_CASE_NUM, DEAL_TEST_COMM_NUM, DEAL_TEST_COMM_COVERAGE, DEAL_PPE15_COVERAGE, \n" + + "DEAL_COEFFICIENT, DEAL_RATE_MEDIAN, THIRD_TEST_CASE_NUM, THIRD_TEST_COMM_NUM, THIRD_TEST_COMM_COVERAGE, THIRD_PPE15_COVERAGE, THIRD_COEFFICIENT, THIRD_RATE_MEDIAN, \n" + + "to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from avm_evaluation_gj where YEAR_MONTH = ${yearMonth}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.YEAR_MONTH + data << r.COMMUNITY_NUM + data << r.HAS_PRICE_COMMUNITY_NUM + data << r.PRICE_COVERAGE + data << r.ARTIFICIAL_CALC_NUM + data << r.ARTIFICIAL_CALC_COVERAGE + data << r.SAMPLE_CALC_NUM + data << r.SAMPLE_CALC_COVERAGE + data << r.ANN_CALC_NUM + data << r.ANN_CALC_COVERAGE + data << r.TEST_NUM + data << r.TEST_COMM_NUM + data << r.TEST_COMM_COVERAGE + data << r.PPE15_COVERAGE + data << r.RATE_MEDIAN + data << r.COEFFICIENT + data << r.DEAL_TEST_CASE_NUM + data << r.DEAL_TEST_COMM_NUM + data << r.DEAL_TEST_COMM_COVERAGE + data << r.DEAL_PPE15_COVERAGE + data << r.DEAL_COEFFICIENT + data << r.DEAL_RATE_MEDIAN + data << r.THIRD_TEST_CASE_NUM + data << r.THIRD_TEST_COMM_NUM + data << r.THIRD_TEST_COMM_COVERAGE + data << r.THIRD_PPE15_COVERAGE + data << r.THIRD_COEFFICIENT + data << r.THIRD_RATE_MEDIAN + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" + fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","YEAR_MONTH","COMMUNITY_NUM","HAS_PRICE_COMMUNITY_NUM","PRICE_COVERAGE","ARTIFICIAL_CALC_NUM","ARTIFICIAL_CALC_COVERAGE","SAMPLE_CALC_NUM","SAMPLE_CALC_COVERAGE", + //"ANN_CALC_NUM","ANN_CALC_COVERAGE","TEST_NUM","TEST_COMM_NUM","TEST_COMM_COVERAGE","PPE15_COVERAGE","RATE_MEDIAN","COEFFICIENT","DEAL_TEST_CASE_NUM","DEAL_TEST_COMM_NUM","DEAL_TEST_COMM_COVERAGE","DEAL_PPE15_COVERAGE", + //"DEAL_COEFFICIENT","DEAL_RATE_MEDIAN","THIRD_TEST_CASE_NUM","THIRD_TEST_COMM_NUM","THIRD_TEST_COMM_COVERAGE","THIRD_PPE15_COVERAGE","THIRD_COEFFICIENT","THIRD_RATE_MEDIAN","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false); + println "AVM_EVALUATION_GJNB 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + CcbFtpTestUtils.ftpUpload(null, fileName1, curDate) + println "AVM_EVALUATION_GJNB 上传Ftp完成......" + } + + def exportAVMEvaluationItemGJNB(yearMonth){ + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, CONFIDENCE,TARGET_AVG_PRICE, \n" + + "to_char(DEALDATE,'yyyy-mm-dd hh24:mi:ss') as DEALDATE, UNIT_PRICE, BUILDAREA, FITMENT, HOUSEASPECT, FLOORNUM, FLOORCNT, DATA_TYPE, RATE, PE, \n" + + "to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME from avm_evaluation_item_gj where YEAR_MONTH = ${yearMonth}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.AVG_PRICE + data << r.CALC_TYPE + data << r.CONFIDENCE + data << r.TARGET_AVG_PRICE + data << r.DEALDATE + data << r.UNIT_PRICE + data << r.BUILDAREA + data << r.FITMENT + data << r.HOUSEASPECT + data << r.FLOORNUM + data << r.FLOORCNT + data << r.DATA_TYPE + data << r.RATE + data << r.PE + data << r.CREATETIME + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" + fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","CONFIDENCE","TARGET_AVG_PRICE","DEALDATE","UNIT_PRICE","BUILDAREA","FITMENT","HOUSEASPECT","FLOORNUM","FLOORCNT","DATA_TYPE","RATE","PE","CREATETIME"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false); + println "AVM_EVALUATION_ITEM_GJNB 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + CcbFtpTestUtils.ftpUpload(null, fileName1, curDate) + println "AVM_EVALUATION_ITEM_GJNB 上传Ftp完成......" + } + + def exportCommunityBasePrice(yearMonth){ + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.AVG_PRICE + data << r.CALC_TYPE + data << r.FAKE_RATE + data << r.DIFF_RATE + data << r.CONFI_RATE + data << r.CORR_RATE + data << r.CONFIDENCE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false); + println "COMMUNITY_BASE_PRICE_GJNB 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + CcbFtpTestUtils.ftpUpload(null, fileName1, curDate) + println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......" + } + + def exportCommunityBasePriceGj(yearMonth){ + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.AVG_PRICE + data << r.CALC_TYPE + data << r.FAKE_RATE + data << r.DIFF_RATE + data << r.CONFI_RATE + data << r.CORR_RATE + data << r.CONFIDENCE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "COMMUNITY_BASE_PRICE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "COMMUNITY_BASE_PRICE_GJ 上传Ftp完成......" + } + + def exportCityPrice(month){ + def sql = "select ID, CITYID, CITYNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from city_price_gj where year_month=${month}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.PRICE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def fileName = "/ccb/MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "CITY_PRICE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "CITY_PRICE_GJ 上传Ftp完成......" + } + + def exportRegionPrice(month){ + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from DISTRICT_PRICE_GJ where year_month=${month}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.PRICE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def fileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "DISTRICT_PRICE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "DISTRICT_PRICE_GJ 上传Ftp完成......" + } + + def exportCommunityPdescribeGj(month){ + +// def citys = CcbCommonUtils.citys; +// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] +// +// for(def city : citys) { +// def cityCode = cityCodeDataMap.get(city) + def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month}"; +// def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time in (202110,202111,202112,202201)"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMUNITY_ID + data << r.CITYID + data << r.CITY_NAME + data << r.DISTRICT_CODE + data << r.DISTRICT_NAME + data << r.COMMUNITY_NAME + data << r.PQ_NAME + data << r.BASE_PRICE + data << r.CONFIDENCE_LEVEL + data << r.CNT + data << r.MIN_PRICE + data << r.MEDIANPRICE + data << r.MEANPRICE + data << r.MAX_PRICE + data << r.TIME + data << r.FIRSTDATAT + data << r.THIRDDATA + data << r.RELATIVEERROR + data << r.SKEWNESS + data << r.PQ_PRICE_RANGE + data << r.DISTRICT_PRICE_RANGE + data << r.DESCRIPTION + data << r.PRIV_BASE_PRICE + data << r.IS_ARTTFICIAL + data << r.PRIV_FINAL_PRICE + data << r.CREATE_DATE + data << r.PRICE + data << r.PRICE_SOURCE + data << r.CASE_MD_PRICE + data << r.CASE_AVG_PRICE + data << r.INDEX_LABEL + data << r.MAIN_PROPERTY + data << r.COMMTYPE + data << r.NEW_DEAL_PRICE + data << r.NEW_DEAL_DATE + data << r.PRICE_MOM_RATIO + data << r.MAX_MOM_RATIO + data << r.PRICE_YOY_RATIO + data << r.SATUR_RATIO + data << r.NEIGHBOR_PRICE + allRows << data + }); + println allRows.size() + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); +// } + println "community_p_describe_gj 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "community_p_describe_gj 上传Ftp完成......" + } + + def exportCommCaseDetailGj(month){ + +// def citys = CcbCommonUtils.citys; + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def fileName = "/ccb/MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] + + for(def city : citys){ + def cityCode = cityCodeDataMap.get(city) + def sql = "select ID, CITYID, COMMUNITY_ID, NETLISTING_ID, PRICE_DATE, to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE from comm_case_detail_gj where cityid='${cityCode}' and price_date=${month}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.COMMUNITY_ID + data << r.NETLISTING_ID + data << r.PRICE_DATE + data << r.CREATE_DATE + allRows << data + if(allRows.size()%200000==0){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + allRows.clear() + } + }); + if(allRows.size()>0){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + } + } + + println "comm_case_detail_gj 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "comm_case_detail_gj 上传Ftp完成......" + } + + def exportLastFile(){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def dataCodeFileName = "/ccb/${curDate}.ok" + if(CommonDatautils.isWindow()){ + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok" + } + FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload(null, dataCodeFileName, curDate) + } + + def exportOkFile(){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def dataCodeFileName = "/ccb/${curDate}.ok" + if(CommonDatautils.isWindow()){ + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok" + } + FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload(null, dataCodeFileName, curDate) + } + + def exportInOk(){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + Vector files = CcbFtpTestUtils.listFileNames(curDate) + Iterator fileVec = files.iterator() + def fileRows = [] + while(fileVec.hasNext()){ + ChannelSftp.LsEntry file = fileVec.next() + + if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){ + fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(file.getFilename(), curDate) + } + } + def fileName = "/ccb/${curDate}.ok" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok" + } + FileUtils.createTxtFile(fileRows, fileName, "utf-8", false); + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "统计文件长度ok文件 上传Ftp完成......" + } +} + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy index f8770132c..19bf6561b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy @@ -6,6 +6,7 @@ import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.lixue.util.DBUtils import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject import java.util.concurrent.CopyOnWriteArrayList @@ -21,6 +22,66 @@ class AddressCut extends BaseBatch{ @Override public void process(Object args) { + sameAddr() + } + + def sameAddr() { + def citys = CommonDatautils.findAllCity() + def dataMap = [:] + def index = 0 + for (def c : citys) { + index++ + def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city", "region", "name", "address"]) + + def addrMap = [:] + for (def dist : dists) { + if (dist.address) { + def rets = addrMap.get(dist.address) + if (!rets) { + rets = [] + } + rets << dist + addrMap.put(dist.address, rets) + } + } + dataMap.put(c, addrMap) + println index +" "+c + } + + + def sameAddrList = [] + for(def c : citys) { + def cityMap = dataMap.get(c) + if(cityMap){ + cityMap.each{addr,rets-> + if(rets && rets.size()>1){ + def keys = new HashSet() + for(def d : rets){ + if(d.name){ + def name = d.name.replace("一期","").replace("二期","").replace("三期","").replace("四期","").replace("五期","").replace("六期","").replace("七期","").replace("八期","").replace("九期","").replace("十期","").replace("一区","").replace("二区","").replace("三区","").replace("四区","").replace("五区","").replace("六区","").replace("七区","").replace("八区","").replace("九区","").replace("十区","").replace("A区","").replace("B区","").replace("C区","").replace("D区","") + .replace("十一街区","").replace("十二街区","").replace("十三街区","").replace("十四街区","").replace("十五街区","").replace("十六街区","").replace("十七街区","").replace("十八街区","").replace("十九街区","").replace("一街区","").replace("二街区","").replace("三街区","").replace("四街区","").replace("五街区","").replace("六街区","").replace("七街区","").replace("八街区","").replace("九街区","").replace("十街区","") + .replace("公寓","").replace("别墅","").replace("商住楼","").replace("花园","").replace("南区","").replace("北区","").replace("东区","").replace("西区","") + keys << d.region +"##"+ name + } + } + if(keys && keys.size()>1){ + for(def d : rets) { + if (d.name) { + sameAddrList << [d.city, d.region, d.name, d.address] + } + } + } + } + } + } + } + //ExcelUtils.write("D:\\数据\\小区\\同地址小区.xlsx", sameAddrList, "相同地址小区") + FileUtils.batchCreateFile(sameAddrList, "D:\\数据\\小区\\同地址小区.txt", null, "utf-8", false); + println "完成" + } + + + def addrCut(){ def citys = CommonDatautils.findAllCity() def index = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/SSHHelperTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/SSHHelperTest.groovy new file mode 100644 index 000000000..0e94c9551 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/SSHHelperTest.groovy @@ -0,0 +1,112 @@ +package com.fangjia.script.ccbevaluation.ronghe + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; + +import com.jcraft.jsch.ChannelExec; +import com.jcraft.jsch.JSch; +import com.jcraft.jsch.Session + +import com.jcraft.jsch.*; + + +class SSHHelperTest { + + private Session session = null; + private Channel channel = null; + + private String sftpHost = "192.168.0.86"; + private int sftpPort = 17495; + private String sftpUserName = "ccb"; + private String sftpPassword = "ccb!@#"; + private int timeout = 30000; + /** + * 获取连接 + * @return + */ + private ChannelExec getChannelExec() { + try { + if (channel != null && channel.isConnected()) { + return (ChannelExec) channel; + } + JSch jSch = new JSch(); + if (session == null || !session.isConnected()) { + session = jSch.getSession(sftpUserName, sftpHost, sftpPort); + session.setPassword(sftpPassword); + Properties config = new Properties(); + config.put("StrictHostKeyChecking", "no"); + session.setConfig(config); + session.setTimeout(timeout); + session.connect(); + } + channel = session.openChannel("exec"); + } catch (Exception e) { + if (session != null) { + session.disconnect(); + session = null; + } + channel = null; + } + return channel == null ? null : (ChannelExec) channel; + } + + /** + * 关闭连接 + */ + private void closeChannel() { + try { + if (channel != null) { + channel.disconnect(); + channel = null; + } + if (session != null) { + session.disconnect(); + session = null; + } + } catch (Exception e) { + System.out.println(e); + } + } + + /** + * 执行服务器端命令 + */ + public boolean executeCommand(String command) { + boolean flag = false; + ChannelExec channelExec = getChannelExec(); + if (channelExec == null) { + return false; + } + try { + channelExec.setInputStream(null); + channelExec.setErrStream(System.err); + channelExec.setCommand(command); + + InputStream ins = channelExec.getInputStream(); + channelExec.connect(); + + BufferedReader reader = new BufferedReader(new InputStreamReader(ins, "utf-8")); + String line = null; + while ((line = reader.readLine()) != null) { + System.out.println(line); + } + + reader.close(); + closeChannel(); + + flag = true; + } catch (Exception e) { + System.out.println(e); + flag = false; + } + return flag; + } + + public static void main(String[] args) { + SSHHelperTest jschCommand = new SSHHelperTest(); + System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/package.sh 20220110")); + } + + +} From 7871c67c82ff8fb87f73bdd867b5c5e31a3dd385 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 28 Oct 2022 16:24:59 +0800 Subject: [PATCH 12/15] update --- .../ccbevaluation/ExportOracleDataTest.groovy | 64 +++++++++---------- .../script/gj/paimai/PaimaiDataSend.groovy | 10 +-- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy index a93ceb174..945ab6a17 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy @@ -32,41 +32,41 @@ class ExportOracleDataTest { def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") def new_dists = [] //不会变 10.25 - exportEvaluationRateData() - exportBigdataPoiData()// - exportBigdataDistrictData()// - - exportGuidedPrice()//随人工改变// - exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策// - - //处理小区情况 - exportBigdataCommunity()// - exportCommunityStaging()//没cityid - exportCommunityBuileType()// - - exportThirdPartyPrice2(lastMonth, new_dists)// - exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 - exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 - - //第三方网站权重表 - exportWebsiteRate(); - - //计算结果相关 - exportAvmEvaluationGJNB(curMonth)//无城市区分 - exportAVMEvaluationItemGJNB(curMonth)//无城市区分 - - exportCityPrice(curMonth)//无城市区分 - exportRegionPrice(curMonth)//无城市区分 - - exportCommunityBasePriceGj(curMonth)//已合下面的合并 //无城市区分 - exportCommunityBasePrice(curMonth)//无城市区分 - - exportCommunityPdescribeGj(curMonth)//无城市区分 - exportCommCaseDetailGj(curMonth)//无城市区分 +// exportEvaluationRateData() +// exportBigdataPoiData()// +// exportBigdataDistrictData()// +// +// exportGuidedPrice()//随人工改变// +// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策// +// +// //处理小区情况 +// exportBigdataCommunity()// +// exportCommunityStaging()//没cityid +// exportCommunityBuileType()// +// +// exportThirdPartyPrice2(lastMonth, new_dists)// +// exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 +// exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 +// +// //第三方网站权重表 +// exportWebsiteRate(); +// +// //计算结果相关 +// exportAvmEvaluationGJNB(curMonth)//无城市区分 +// exportAVMEvaluationItemGJNB(curMonth)//无城市区分 +// +// exportCityPrice(curMonth)//无城市区分 +// exportRegionPrice(curMonth)//无城市区分 +// +// exportCommunityBasePriceGj(curMonth)//已合下面的合并 //无城市区分 +// exportCommunityBasePrice(curMonth)//无城市区分 +// +// exportCommunityPdescribeGj(curMonth)//无城市区分 +// exportCommCaseDetailGj(curMonth)//无城市区分 def curDate = curMonth.toString()+"28" // ok文件 -// exportInOk() + exportInOk() SSHHelperTest jschCommand = new SSHHelperTest(); System.out.println(jschCommand.executeCommand("sh /home/ccb/output/eval_package.sh ${curDate}")); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy index 83ceb7171..cfc3ad6d2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy @@ -38,8 +38,8 @@ class PaimaiDataSend { // task.jdRunByMongo() // task.aliZcRun() - task.aliZcRun2() -// task.aliFpRun() +// task.aliZcRun2() + task.aliFpRun() // task.rmfysszcRun() // task.jdRun() // task.checkUrl() @@ -334,10 +334,10 @@ class PaimaiDataSend { def index = 0 def q = new BasicDBObject() // q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"])) - q.put("province","江苏") - q.put("sub_type","土地") +// q.put("province","江苏") +// q.put("sub_type","土地") // q.put("parser_status", new BasicDBObject('$ne',2)) - q.put("url","https://susong-item.taobao.com/auction/671320162557.htm") + q.put("url","https://sf-item.taobao.com/sf_item/673646300374.htm") //https://sf-item.taobao.com/sf_item/657131607486.htm // q.put("parser_status", new BasicDBObject('$exists',false)) // q.put("finishDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-09-01"))) From 57ca8daf07787db4666a50dc6351f4604f521372 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 28 Oct 2022 17:13:28 +0800 Subject: [PATCH 13/15] =?UTF-8?q?=E7=9B=AE=E5=BD=95=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/ExportOracleDataTest.groovy | 56 +++++++++---------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy index 945ab6a17..8a8abaf10 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy @@ -68,7 +68,7 @@ class ExportOracleDataTest { // ok文件 exportInOk() SSHHelperTest jschCommand = new SSHHelperTest(); - System.out.println(jschCommand.executeCommand("sh /home/ccb/output/eval_package.sh ${curDate}")); + System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}")); } @@ -91,7 +91,7 @@ class ExportOracleDataTest { }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; - def fileName = "/ccb/MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" } @@ -122,7 +122,7 @@ class ExportOracleDataTest { }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; - def fileName = "/ccb/MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat" } @@ -157,7 +157,7 @@ class ExportOracleDataTest { }); } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; - def fileName = "/ccb/MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" } @@ -185,7 +185,7 @@ class ExportOracleDataTest { allRows << data }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; - def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" } @@ -220,7 +220,7 @@ class ExportOracleDataTest { def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; - def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" } @@ -259,7 +259,7 @@ class ExportOracleDataTest { } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; - def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" } @@ -299,7 +299,7 @@ class ExportOracleDataTest { } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; - def fileName = "/ccb/MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" } @@ -336,7 +336,7 @@ class ExportOracleDataTest { } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; - def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" } @@ -381,7 +381,7 @@ class ExportOracleDataTest { } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; - def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" } @@ -502,7 +502,7 @@ class ExportOracleDataTest { allRows << data }) def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; - def fileName = "/ccb/MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" } @@ -519,7 +519,7 @@ class ExportOracleDataTest { def exportThirdPartyPrice2(price_date, new_dists) { def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; - def fileName = "/ccb/MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" // fileName = "D:\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" @@ -627,7 +627,7 @@ class ExportOracleDataTest { } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; - def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" } @@ -643,7 +643,7 @@ class ExportOracleDataTest { def exportBigdataForsale(publishDate, new_dists){ //["北京","上海"] def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; - def fileName = "/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" } @@ -809,8 +809,8 @@ class ExportOracleDataTest { }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; - def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" - def fileName1 = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = "/ccb_evaluation/MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" @@ -861,8 +861,8 @@ class ExportOracleDataTest { }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; - def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" - def fileName1 = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = "/ccb_evaluation/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat" @@ -904,8 +904,8 @@ class ExportOracleDataTest { }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; - def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" - def fileName1 = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" @@ -947,7 +947,7 @@ class ExportOracleDataTest { }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; - def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" } @@ -976,7 +976,7 @@ class ExportOracleDataTest { }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" - def fileName = "/ccb/MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat" } @@ -1007,7 +1007,7 @@ class ExportOracleDataTest { }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" - def fileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat" } @@ -1023,7 +1023,7 @@ class ExportOracleDataTest { // def citys = CcbCommonUtils.citys; // def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" - def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" } @@ -1093,7 +1093,7 @@ class ExportOracleDataTest { def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" - def fileName = "/ccb/MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" } @@ -1129,7 +1129,7 @@ class ExportOracleDataTest { def exportLastFile(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" - def dataCodeFileName = "/ccb/${curDate}.ok" + def dataCodeFileName = "/ccb_evaluation/${curDate}.ok" if(CommonDatautils.isWindow()){ dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok" } @@ -1139,7 +1139,7 @@ class ExportOracleDataTest { def exportOkFile(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" - def dataCodeFileName = "/ccb/${curDate}.ok" + def dataCodeFileName = "/ccb_evaluation/${curDate}.ok" if(CommonDatautils.isWindow()){ dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok" } @@ -1159,7 +1159,7 @@ class ExportOracleDataTest { fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(file.getFilename(), curDate) } } - def fileName = "/ccb/${curDate}.ok" + def fileName = "/ccb_evaluation/${curDate}.ok" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok" } From 2122bf384760d4b9860ef240ed34e483f50c09a0 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 28 Oct 2022 17:20:18 +0800 Subject: [PATCH 14/15] update --- .../ccbevaluation/CcbFtpTestUtils.groovy | 2 +- .../ccbevaluation/ExportOracleDataTest.groovy | 62 +++++++++---------- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy index b9f4572e5..2a9ad549f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy @@ -11,7 +11,7 @@ class CcbFtpTestUtils { static def ftpPort = 17495 static def ftpUser = "ccb" static def ftpPassword = "ccb!@#" - static def ftpPath = "/home/ccb/output/" + static def ftpPath = "/home/ccb_evaluation/output/" static def ftpPath_rh = "/home/ccb_rh/output/" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy index 8a8abaf10..431fc02c2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy @@ -32,37 +32,37 @@ class ExportOracleDataTest { def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") def new_dists = [] //不会变 10.25 -// exportEvaluationRateData() -// exportBigdataPoiData()// -// exportBigdataDistrictData()// -// -// exportGuidedPrice()//随人工改变// -// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策// -// -// //处理小区情况 -// exportBigdataCommunity()// -// exportCommunityStaging()//没cityid -// exportCommunityBuileType()// -// -// exportThirdPartyPrice2(lastMonth, new_dists)// -// exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 -// exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 -// -// //第三方网站权重表 -// exportWebsiteRate(); -// -// //计算结果相关 -// exportAvmEvaluationGJNB(curMonth)//无城市区分 -// exportAVMEvaluationItemGJNB(curMonth)//无城市区分 -// -// exportCityPrice(curMonth)//无城市区分 -// exportRegionPrice(curMonth)//无城市区分 -// -// exportCommunityBasePriceGj(curMonth)//已合下面的合并 //无城市区分 -// exportCommunityBasePrice(curMonth)//无城市区分 -// -// exportCommunityPdescribeGj(curMonth)//无城市区分 -// exportCommCaseDetailGj(curMonth)//无城市区分 + exportEvaluationRateData() + exportBigdataPoiData()// + exportBigdataDistrictData()// + + exportGuidedPrice()//随人工改变// + exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策// + + //处理小区情况 + exportBigdataCommunity()// + exportCommunityStaging()//没cityid + exportCommunityBuileType()// + + exportThirdPartyPrice2(lastMonth, new_dists)// + exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 + exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 + + //第三方网站权重表 + exportWebsiteRate(); + + //计算结果相关 + exportAvmEvaluationGJNB(curMonth)//无城市区分 + exportAVMEvaluationItemGJNB(curMonth)//无城市区分 + + exportCityPrice(curMonth)//无城市区分 + exportRegionPrice(curMonth)//无城市区分 + + exportCommunityBasePriceGj(curMonth)//已合下面的合并 //无城市区分 + exportCommunityBasePrice(curMonth)//无城市区分 + + exportCommunityPdescribeGj(curMonth)//无城市区分 + exportCommCaseDetailGj(curMonth)//无城市区分 def curDate = curMonth.toString()+"28" // ok文件 From 110118291e34355de6d8f3205be1e943f34c970a Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Fri, 28 Oct 2022 18:45:01 +0800 Subject: [PATCH 15/15] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=AD=A6=E6=A0=A1?= =?UTF-8?q?=E7=BB=8F=E7=BA=AC=E5=BA=A6=E8=8E=B7=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/poi/FillLngLat.groovy | 20 ++++---- .../temp/shenjian/util/BaiDuUtils.groovy | 48 +++++++++++++++++++ 2 files changed, 59 insertions(+), 9 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/BaiDuUtils.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/FillLngLat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/FillLngLat.groovy index a0c3ec24f..3c5c2a52d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/FillLngLat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/FillLngLat.groovy @@ -1,6 +1,7 @@ package com.fangjia.script.temp.shenjian.poi import com.alibaba.fastjson.JSONObject +import com.fangjia.script.temp.shenjian.util.BaiDuUtils import com.fangjia.script.temp.shenjian.util.EsUtils import com.fangjia.script.temp.shenjian.util.GaoDeUtils import com.fangjia.script.util.ElasticUtils @@ -15,25 +16,26 @@ import java.util.concurrent.Executors */ class FillLngLat { - static ExecutorService executorService = Executors.newFixedThreadPool(4) +// static ExecutorService executorService = Executors.newFixedThreadPool(4) static void main(String[] args) { - ElasticUtils.executeSqlQuery("select * from dws_poi where location is not null", { dataT -> + ElasticUtils.executeSqlQuery("select * from dws_poi where location is null", { dataT -> final def data = dataT - executorService.submit(new Runnable() { - @Override - void run() { +// executorService.submit(new Runnable() { +// @Override +// void run() { println(data) - def lngLat = GaoDeUtils.parserByGaode(data.city, data.region + data.name + data.address) + def lngLat = BaiDuUtils.parserByBaidu(data.city, data.region + data.name + data.address) + Thread.sleep(1000) if (lngLat != null && lngLat.size() == 2 ) { JSONObject jsonObject = new JSONObject() jsonObject.put("lon", Double.valueOf(lngLat['lng'])) jsonObject.put("lat", Double.valueOf(lngLat['lat'])) EsUtils.bulkUpdateFieldByIds("dws_poi", "location", jsonObject, Arrays.asList(data.source_id)) } - } - }) +// } +// }) }) - executorService.shutdown() + //executorService.shutdown() } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/BaiDuUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/BaiDuUtils.groovy new file mode 100644 index 000000000..e307bf28a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/BaiDuUtils.groovy @@ -0,0 +1,48 @@ +package com.fangjia.script.temp.shenjian.util + +import com.alibaba.fastjson.JSONObject +import com.fangjia.core.util.GeoUtils +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.util.DataUtil +import org.apache.commons.lang.StringUtils + +/** + * 百度工具类 + * + * @author shenjian + * @since 2022/10/28 + */ +class BaiDuUtils { + + def static final String KEY = "6w9SEHS3hnAFjmY2oEhqIDdgvLPKWXu5" + + /** + * 百度地址逆解析,返回高德坐标!!!!!!!!!!!!!!! + * @param city + * @param address + */ + def static parserByBaidu(def city, def address) { + Map params = new HashMap() + params.put("city", city) + if (StringUtils.isBlank(address)) { + return [] + } + params.put("address", address.replaceAll("null", "")) + params.put("output", "json") + params.put("ak", KEY) + params.put("callback", "showLocation") + + String content + content = new HttpRequest().get("https://api.map.baidu.com/geocoding/v3/", params, "utf-8") + //content = "showLocation&&showLocation({\"status\":0,\"result\":{\"location\":{\"lng\":131.04009309945693,\"lat\":45.78426770180344},\"precise\":1,\"confidence\":70,\"comprehension\":100,\"level\":\"地产小区\"}})" + content = content.replaceAll("showLocation&&showLocation\\(", "").replaceAll("\\)", "") + JSONObject json = JSONObject.parseObject(content) + if (json.status == 0 && json.result && json.result.location) { + def bd_lng = json.result.location.lng + def bd_lat = json.result.location.lat + Map locationMap = GeoUtils.bd09ToGcj02(bd_lat, bd_lng) + return ["lng": com.fangjia.core.util.DataUtil.getDoubleRound(6, locationMap['lng']), "lat": com.fangjia.core.util.DataUtil.getDoubleRound(6, locationMap['lat'])] + } + return [] + } +}