From 74c1bb4b998f3032ee1b995c9987e4152f692ead Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 25 Oct 2022 13:44:12 +0800 Subject: [PATCH 001/112] =?UTF-8?q?=E5=AF=BC=E5=87=BA=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/ExportOracleData.groovy | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index e1e17ee47..462d76787 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -25,7 +25,7 @@ class ExportOracleData { def run(args) { citys = CcbCommonUtils.getRunningCitys(args) -// def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); + def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") def new_dists = [] @@ -41,7 +41,7 @@ class ExportOracleData { exportBigdataCommunity()// exportCommunityStaging()//没cityid exportCommunityBuileType()// - +// exportThirdPartyPrice2(lastMonth, new_dists)// exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 @@ -51,23 +51,23 @@ class ExportOracleData { //计算结果相关 -// exportAvmEvaluationGJNB(curMonth) -// exportAVMEvaluationItemGJNB(curMonth) -// -// exportCityPrice(curMonth) -// exportRegionPrice(curMonth) -// -// exportCommunityBasePriceGj(curMonth)//已合下面的合并 -// exportCommunityBasePrice(curMonth) -// -// exportCommunityPdescribeGj(curMonth) -// exportCommCaseDetailGj(curMonth) + exportAvmEvaluationGJNB(curMonth) + exportAVMEvaluationItemGJNB(curMonth) -// def curDate = curMonth.toString()+"28" -// SSHHelper jschCommand = new SSHHelper(); -// System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}")); - //ok文件 -// exportLastFile() + exportCityPrice(curMonth) + exportRegionPrice(curMonth) + + exportCommunityBasePriceGj(curMonth)//已合下面的合并 + exportCommunityBasePrice(curMonth) + + exportCommunityPdescribeGj(curMonth) + exportCommCaseDetailGj(curMonth) + + def curDate = curMonth.toString()+"28" + SSHHelper jschCommand = new SSHHelper(); + System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}")); +// ok文件 + exportLastFile() } @@ -376,7 +376,7 @@ class ExportOracleData { "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + "from bigdata_deal_gj where CITYID='${cityCode}' and dealdate>=to_date('${dealDate}','yyyy-MM-dd')"; - allRows = dealData(sql) + allRows.addAll(dealData(sql)) } } From eaf35c26ac74cc43bff7a8a3ba5be88e6f6c6517 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 25 Oct 2022 13:44:46 +0800 Subject: [PATCH 002/112] =?UTF-8?q?=E7=AC=AC=E4=B8=89=E6=96=B9=E4=BF=AE?= =?UTF-8?q?=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../thirdPriceCrawl/DataStandardBatch.groovy | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index 91d3dee7e..7c1ad2c31 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -91,11 +91,9 @@ class DataStandardBatch extends BaseBatch{ // ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl) cityMap.each{city,alias-> -// if(city == "北京"){ - println "开始加载【${city}】" - def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"]) - districtMap.putAll(detailMap) -// } + println "开始加载【${city}】" + def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"]) + districtMap.putAll(detailMap) } while(flag){ @@ -122,9 +120,9 @@ class DataStandardBatch extends BaseBatch{ def final AtomicInteger count_ = new AtomicInteger(0) def final AtomicInteger successIndex = new AtomicInteger(0) datas.each { dbo -> - fixedThreadPool.execute( - new Runnable() { - void run() { +// fixedThreadPool.execute( +// new Runnable() { +// void run() { try{ println "(${rowIndex}-${count}-${count_.incrementAndGet()})【${dbo.city}-${dbo.region}-${dbo.name}-${dbo.address}-${dbo.source}】" if(dbo.name){ @@ -164,12 +162,12 @@ class DataStandardBatch extends BaseBatch{ }finally { latch.countDown() } - } - } - ); +// } +// } +// ); } - latch.await() +// latch.await() long end = System.currentTimeMillis() println "第【${rowIndex}】次中匹配成功的:"+successCount+" 个,耗时:"+ (end - start)+" "+ DateUtils.format(new Date(),"yyyy-MM-dd HH:mm:ss") @@ -502,6 +500,8 @@ class DataStandardBatch extends BaseBatch{ type = 12;//区域不同 district = null; }else{ + def score = CosineSimilarAlgorithm.getSimilarity(district.name, copyObj.name) + district.score = score //此处用来做测试 type=15 } } @@ -922,7 +922,7 @@ class DataStandardBatch extends BaseBatch{ qdata.put("s_date", SHORT_DATE) // qdata.put("city","北京") // qdata.put("source","安居客") -// qdata.put("name",Pattern.compile("金隅澜湾.*")) +// qdata.put("name",Pattern.compile("龚家宅.*")) qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false)); def ret = [] try{ @@ -1065,6 +1065,9 @@ class DataStandardBatch extends BaseBatch{ udata2.put("match_region", district.region); udata2.put("match_name", district.name); udata2.put("match_address", district.address); + if(district.score){ + udata2.put("score", district.score); + } udata2.put("avg_price", data.avg_price); udata2.put("type", type); udata2.put("tid", data._id.toString()); From 17f7722530be18c2f63f70d36a893ea28dc95010 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 25 Oct 2022 13:45:44 +0800 Subject: [PATCH 003/112] update --- .../ccbevaluation/BigdataDealBatch.groovy | 2 +- .../ccbevaluation/GuiDingPriceBatch.groovy | 4 +-- .../fangjia/script/ccbevaluation/Test.groovy | 28 +++++++++---------- .../script/gj/chongqingblank/DataMatch.groovy | 25 ++++++++++++++++- .../fangjia/script/util/OpenApiClient.groovy | 16 +++++++++++ 5 files changed, 57 insertions(+), 18 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 89b9f7f35..82972cdd3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -58,7 +58,7 @@ class BigdataDealBatch extends BaseBatch{ public void process(Object args) { //def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date()) // def citys = CcbCommonUtils.citys; - def citys = CcbCommonUtils.getRunningCitys(args) + def citys = ["上海"]//CcbCommonUtils.getRunningCitys(args) if(args && args.size()>=2){ curDate = args[1].toString().toInteger() } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 936022f98..c390e6c83 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -36,7 +36,7 @@ class GuiDingPriceBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + def citys = CcbCommonUtils.getRunningCitys(args) def rows = [] for (def city : citys) { @@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2022-09-22", "2022-09-23") + def prices = CcbCommonUtils.findPersonPrice(city, "2022-10-21", "2022-10-22") if(prices){ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 31ae30039..e987ea5af 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -1001,10 +1001,10 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business -//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\录入人工查价_0922_300.xlsx" +def fileName = "D:\\数据\\建行\\房产估值\\指导价\\人工指导价录入_1020.xlsx" -//ExcelUtils.multiThreadingModify(fileName, {d-> -// if(d['A']){ +ExcelUtils.multiThreadingModify(fileName, {d-> + if(d['A']){ // def region = RegionData.formatRegion(d['A'], d['B']) // if(region){ // def dist = CommonDatautils.findDistrict(d['A'], region, d['C'], "district") @@ -1013,18 +1013,18 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business // d['F'] = dist._id.toString() // } // } -//// if(!d['Q']){ -//// def dist = CommonDatautils.findDistrictById(d['A']) -////// -//// if(dist){ -//// d['H'] = dist.region -////// d['G'] = dist.name -//// d['I'] = dist.status -//// } -//// } + if(!d['F']){ + def dist = CommonDatautils.findDistrictById(d['A']) // -// } -//},0) + if(dist){ +// d['H'] = dist.region +// d['G'] = dist.name + d['F'] = dist.status + } + } + + } +},0) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy index 418b6c102..bba57a816 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy @@ -29,7 +29,30 @@ class DataMatch { // task.run3() // println task.findAddress("昌州街道棠香路") - task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx") +// task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx") + task.fzWeiyuanshuMatch("D:\\数据\\福州\\商品住房楼盘评估指导参考价情况表_匹配.xlsx") + } + + def fzWeiyuanshuMatch(fileName){ + ExcelUtils.modify(fileName,{d-> + if(!d['K'] && d['C']){ + def arr = OpenApiClient.search("福州", d['G'], d['C']) + if(arr){ + d['H'] = arr.city + d['I'] = arr.district + d['J'] = arr.name + } + } + +// def results = null +// if(!d['K'] && d['C']){ +// results = OpenApiClient.addressMatch("福州",d['C'], "property", false, "长乐", null) +// } +// if(results && results.precise>=0){ +// d['L'] = results.district +// d['M'] = results.name +// } + },0) } def run1(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index 68cd266c8..cc7b1ad50 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -52,6 +52,22 @@ public class OpenApiClient { return false } + public static def search(def city, def district, def name){ + Map params = new HashMap() + params.put("city", city); + params.put("keyword", name); + params.put("district", district) + params.put("token", TOKEN); + String content = new HttpRequest().get("http://open.fangjia.com/property/search", params, "utf-8") + JSONObject jsonObj = JSONObject.fromObject(content) + JSONArray array = jsonObj.get("result") + def matchObj = null + if(array){ + matchObj = array[0] + } + return matchObj + } + public static List searchDistrict(double lng, double lat, int r, String city) { DecimalFormat format = new DecimalFormat("#.0000000000"); try { From b137e0f08fbc3edbff3eadf49b0fb7441ecea2f8 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 26 Oct 2022 10:59:26 +0800 Subject: [PATCH 004/112] =?UTF-8?q?=E5=8E=BB=E6=8E=89=E4=BA=86=E4=B8=80?= =?UTF-8?q?=E4=BA=9B=E7=8E=B0=E5=9C=A8=E4=B8=8D=E9=9C=80=E8=A6=81=E7=9A=84?= =?UTF-8?q?=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ThirdDataToGatherBatch.groovy | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy index 3efe71b67..850ef9d42 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy @@ -81,8 +81,8 @@ class ThirdDataToGatherBatch extends BaseBatch{ println "小区添加价格完成" districts = addOtherWebsitePrice(districts); println "小区添加第三方价格完成" - districts = addLevel(districts); - println "小区评级完成" +// districts = addLevel(districts); +// println "小区评级完成" saveOrUpdate(districts); println city + "保存修改完成" toLineDB(city) @@ -101,16 +101,16 @@ class ThirdDataToGatherBatch extends BaseBatch{ def tempPrice = d.get(field+"_price") if(!tempPrice){ d.put(field+"_price", m.avg_price.toInteger()) - if(d.fangjia_price && m.avg_price){ - if(d.fangjia_price){ - try{ - d.put(field+"_diff", diffPercent(d.fangjia_price.toInteger(), m.avg_price.toInteger())) - }catch(Exception e){ - println "dddd" - } - - } - } +// if(d.fangjia_price && m.avg_price){ +// if(d.fangjia_price){ +// try{ +// d.put(field+"_diff", diffPercent(d.fangjia_price.toInteger(), m.avg_price.toInteger())) +// }catch(Exception e){ +// println "dddd" +// } +// +// } +// } } }else{ @@ -208,8 +208,9 @@ class ThirdDataToGatherBatch extends BaseBatch{ query.put("s_date", SHORT_DATE) DBObject udata = new BasicDBObject() - udata.put("level",d.level); - udata.put("last_price", d.last_price); +// udata.put("level",d.level); +// udata.put("last_price", d.last_price); + udata.put("district_id", d._id.toString()) udata.put("fangjia_price", d.fangjia_price); fieldMap.each{k,v-> From da38bead89c598b7f7760707d1770ee17f862cd8 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 26 Oct 2022 17:35:13 +0800 Subject: [PATCH 005/112] =?UTF-8?q?=E5=BB=BA=E8=A1=8C=E7=9B=B8=E5=85=B3?= =?UTF-8?q?=E8=84=9A=E6=9C=AC=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../BigdataCommunityBatch.groovy | 43 +++++++------------ .../ccbevaluation/BigdataForsaleBatch.groovy | 1 - .../script/ccbevaluation/CcbFtpUtils.groovy | 8 ++++ .../ccbevaluation/ExportOracleData.groovy | 38 ++++++++-------- .../gj/district/FangjiaClientUtils.groovy | 17 ++++++++ .../fangjia/script/jiangnanblank/SFTP.groovy | 16 +++++++ .../ThirdDataToGatherBatch.groovy | 2 +- .../fangjia/script/util/OpenApiClient.groovy | 1 + 8 files changed, 77 insertions(+), 49 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index c06cef98e..a96bb9b09 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -8,6 +8,7 @@ import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.ccb.CcbCodeUtils import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.OpenApiClient @@ -175,38 +176,25 @@ class BigdataCommunityBatch extends BaseBatch{ ret << "pa101" ret << "" if(dist.address){ - if(dist.address.toString().contains("弄")){ - String number = DataUtil.match(dist.address, Pattern.compile("\\d+弄")) - if(number){ - ret << dist.address.toString().substring(0, dist.address.toString().indexOf(number)) - ret << dist.address.toString().substring(dist.address.toString().indexOf(number), dist.address.toString().size()) + def jsonObj = null + if(dist.address){ + jsonObj = FangjiaClientUtils.addressStructure(dist.city, dist.address) + } + if(jsonObj && jsonObj.street){ + if(jsonObj.street){ + ret << jsonObj.street }else{ ret << "" + } + + if(jsonObj.streetNumber){ + ret << jsonObj.streetNumber + }else{ ret << "" } }else{ - def jsonObj = null - if(dist.address){ -// jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name) - jsonObj = OpenApiClient.standard1Action(dist.city, dist.address) - } - if(jsonObj && jsonObj.street){ - if(jsonObj.street){ - ret << jsonObj.street - }else{ - ret << "" - } - - if(jsonObj.streetNumber){ - ret << jsonObj.streetNumber - }else{ - ret << "" - } - - }else{ - ret << "" - ret << "" - } + ret << "" + ret << "" } }else{ ret << "" @@ -225,7 +213,6 @@ class BigdataCommunityBatch extends BaseBatch{ println dist.city + " "+dist.name+" error" e.printStackTrace() } - return ret; } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index ca517f31a..5b0edbae1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -448,7 +448,6 @@ class BigdataForsaleBatch extends BaseBatch{ if(!alias.contains(newName)){ alias << newName } - } if(alias && alias.size()>0){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy index 6b972553e..124911f34 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy @@ -3,6 +3,7 @@ package com.fangjia.script.ccbevaluation import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.jiangnanblank.SFTP import com.jcraft.jsch.ChannelSftp +import com.jcraft.jsch.SftpATTRS import org.apache.commons.codec.digest.DigestUtils class CcbFtpUtils { @@ -64,6 +65,13 @@ class CcbFtpUtils { return files } + static def getFileLength(String path, String month){ + sftpChannel = reConnect(); + def size = sftp.getFileSize(ftpPath+ month+"/"+path, sftpChannel) + return size + } + + static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index eabf59e2a..22861ba1d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -30,24 +30,24 @@ class ExportOracleData { def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") def new_dists = [] //不会变 10.25 -// exportEvaluationRateData() -// exportBigdataPoiData()// -// exportBigdataDistrictData()// -// -// exportGuidedPrice()//随人工改变// -// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策// -// -//// //处理小区情况 -// exportBigdataCommunity()// -// exportCommunityStaging()//没cityid + exportEvaluationRateData() + exportBigdataPoiData()// + exportBigdataDistrictData()// + + exportGuidedPrice()//随人工改变// + exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策// + +// //处理小区情况 + exportBigdataCommunity()// + exportCommunityStaging()//没cityid exportCommunityBuileType()// -// -// exportThirdPartyPrice2(lastMonth, new_dists)// -// exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 -// exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 -// -// //第三方网站权重表 -// exportWebsiteRate(); + + exportThirdPartyPrice2(lastMonth, new_dists)// + exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 + exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 + + //第三方网站权重表 + exportWebsiteRate(); // // // //计算结果相关 @@ -229,8 +229,8 @@ class ExportOracleData { FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "BIGDATA_COMMBUILDTYPE_GJ 数据文件输出完成......" -// CcbFtpUtils.ftpUpload(null, fileName, curDate) -// println "BIGDATA_COMMBUILDTYPE_GJ 上传Ftp完成......" + CcbFtpUtils.ftpUpload(null, fileName, curDate) + println "BIGDATA_COMMBUILDTYPE_GJ 上传Ftp完成......" } def exportBigdataDistrictData() { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 9f3670a44..ae1dee2e4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -67,6 +67,7 @@ class FangjiaClientUtils { + //分词 省,市,区 public static def addressStand(def city, def address){ def jsonObj = null def content = null @@ -84,6 +85,22 @@ class FangjiaClientUtils { return jsonObj; } + //分词基础上的标准化,complete=false 不不全 short=true 省市区返回简拼 + public static def addressStructure(def city, def address){ + def jsonObj = null + def content = null + try { + address = address.replace("{","") + def url = "http://192.168.0.19:31080/FangJiaAddressService/structure/v2?address="+URLEncoder.encode(address,"UTF-8")+"&city="+URLEncoder.encode(city,"UTF-8") + content = new HttpRequest().newGet(url, null, "utf-8") + jsonObj = JSONObject.fromObject(content) + } catch (Exception e) { + println address +" "+content + e.printStackTrace(); + } + return jsonObj; + } + public static def arroundPrice(city,address){ def jsonObj = null def content = null diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy index 70082d882..a50827e27 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy @@ -209,4 +209,20 @@ class SFTP { return isDirExistFlag; } + public long getFileSize(String srcSftpFilePath, ChannelSftp sftp) { + long fileSize;//文件大于等于0则存在 + try { + SftpATTRS sftpATTRS = sftp.lstat(srcSftpFilePath); + fileSize = sftpATTRS.getSize(); + } catch (Exception e) { + e.printStackTrace() + fileSize = -1;//获取文件大小异常 + if (e.getMessage().toLowerCase().equals("no such file")) { + fileSize = -2;//文件不存在 + } + } + return fileSize; + } + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy index 850ef9d42..320e65b2f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy @@ -210,7 +210,7 @@ class ThirdDataToGatherBatch extends BaseBatch{ DBObject udata = new BasicDBObject() // udata.put("level",d.level); // udata.put("last_price", d.last_price); - udata.put("district_id", d._id.toString()) + udata.put("seaweed_id", d._id.toString()) udata.put("fangjia_price", d.fangjia_price); fieldMap.each{k,v-> diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index cc7b1ad50..7c37cae97 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -364,6 +364,7 @@ public class OpenApiClient { } + //此接口地址标准化,带补全 public static JSONObject standard1Action(String city, String address) { try { Map params = new HashMap(); From 5873c62bab26abcc9c744955d22b6c6e65000be7 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 27 Oct 2022 14:17:13 +0800 Subject: [PATCH 006/112] update --- .../BigdataCommunityBatch.groovy | 13 ++++--- .../ccbevaluation/CcbFtpTestUtils.groovy | 16 +++++++++ .../ExportOracleEvaluationData.groovy | 9 ++--- .../ccbevaluation/GuiDingPriceBatch.groovy | 2 +- .../script/gj/chongqingblank/DataMatch.groovy | 36 +++++++++---------- 5 files changed, 46 insertions(+), 30 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index a96bb9b09..fafe61ba9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -36,7 +36,7 @@ class BigdataCommunityBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.citys_92; + def citys = CcbCommonUtils.getRunningCitys(args) def rows = [] @@ -108,7 +108,6 @@ class BigdataCommunityBatch extends BaseBatch{ if(ret){ rets << ret } - } catch (Exception e) { e.printStackTrace() } finally { @@ -180,15 +179,15 @@ class BigdataCommunityBatch extends BaseBatch{ if(dist.address){ jsonObj = FangjiaClientUtils.addressStructure(dist.city, dist.address) } - if(jsonObj && jsonObj.street){ - if(jsonObj.street){ - ret << jsonObj.street + if(jsonObj && jsonObj.result && jsonObj.result.street){ + if(jsonObj.result.street){ + ret << jsonObj.result.street }else{ ret << "" } - if(jsonObj.streetNumber){ - ret << jsonObj.streetNumber + if(jsonObj.result.streetNumber){ + ret << jsonObj.result.streetNumber }else{ ret << "" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy index 7083cb615..b9f4572e5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy @@ -70,6 +70,22 @@ class CcbFtpTestUtils { } + static def listFileNames(month){ + //上传到ftp + sftpChannel = reConnect(); + Vector files = sftp.listFiles(ftpPath+ month+"/", sftpChannel); + return files + } + + static def getFileLength(String path, String month){ + sftpChannel = reConnect(); + def size = sftp.getFileSize(ftpPath+ month+"/"+path, sftpChannel) + return size + } + + + + static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy index 7258934fa..b37ee5b94 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy @@ -12,13 +12,14 @@ class ExportOracleEvaluationData { def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + def citys = null static main(args) { def task = new ExportOracleEvaluationData() - task.run() + task.run(args) } - def run() { - + def run(args) { + citys = CcbCommonUtils.getRunningCitys(args) def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); //计算结果相关 @@ -369,7 +370,7 @@ class ExportOracleEvaluationData { def exportCommCaseDetailGj(month){ - def citys = CcbCommonUtils.citys; +// def citys = CcbCommonUtils.citys; def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index c390e6c83..6981c7149 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2022-10-21", "2022-10-22") + def prices = CcbCommonUtils.findPersonPrice(city, "2022-10-22", "2022-10-27") if(prices){ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy index bba57a816..cb615604a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy @@ -30,28 +30,28 @@ class DataMatch { // println task.findAddress("昌州街道棠香路") // task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx") - task.fzWeiyuanshuMatch("D:\\数据\\福州\\商品住房楼盘评估指导参考价情况表_匹配.xlsx") + task.fzWeiyuanshuMatch("D:\\数据\\福州\\连江楼盘.xlsx") } def fzWeiyuanshuMatch(fileName){ - ExcelUtils.modify(fileName,{d-> - if(!d['K'] && d['C']){ - def arr = OpenApiClient.search("福州", d['G'], d['C']) - if(arr){ - d['H'] = arr.city - d['I'] = arr.district - d['J'] = arr.name - } - } + ExcelUtils.multiThreadingModify(fileName,{d-> +// if(!d['G']){ +// def arr = OpenApiClient.search("福州", "连江", d['C']) +// if(arr){ +// d['E'] = arr.city +// d['F'] = arr.district +// d['G'] = arr.name +// } +// } -// def results = null -// if(!d['K'] && d['C']){ -// results = OpenApiClient.addressMatch("福州",d['C'], "property", false, "长乐", null) -// } -// if(results && results.precise>=0){ -// d['L'] = results.district -// d['M'] = results.name -// } + def results = null + if(d['H'] && d['H']=="F" && d['C']){ + results = OpenApiClient.addressMatch("福州",d['C'], "property", false, "连江", null) + } + if(results && results.precise>=0){ + d['J'] = results.district + d['K'] = results.name + } },0) } From aa25d429d2db8797ad0ea6ca7f6572ad847501c5 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 27 Oct 2022 14:21:21 +0800 Subject: [PATCH 007/112] delete --- .../src/main/java/com/fangjia/core/util/DataUtil.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java b/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java index adfb2c7c8..4aaabb88f 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java @@ -25,8 +25,6 @@ import java.util.zip.GZIPOutputStream; import com.hankcs.hanlp.HanLP; import com.hankcs.hanlp.dictionary.py.Pinyin; -import com.sun.javafx.scene.transform.TransformUtils; -import net.sf.json.JSON; import net.sf.json.JSONArray; import net.sf.json.JSONNull; import net.sf.json.JSONObject; From 22c2852dcd1d53e2911ede14ef5b11c6a5b19693 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 27 Oct 2022 16:37:02 +0800 Subject: [PATCH 008/112] =?UTF-8?q?=E6=89=B9=E9=87=8F=E5=9C=B0=E5=9D=80?= =?UTF-8?q?=E6=8B=86=E5=88=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/data/AddressCut.groovy | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy new file mode 100644 index 000000000..e34c2b4af --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy @@ -0,0 +1,80 @@ +package com.fangjia.script.ccbevaluation.data + +import com.fangjia.script.BaseBatch +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.DbUtils +import com.mongodb.BasicDBObject + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class AddressCut extends BaseBatch{ + static main(args) { + runBatch(new AddressCut(), args); + } + + @Override + public void process(Object args) { + def citys = CommonDatautils.findAllCity() + def index = 0 + + def fileName = "/home/goojia/gj/address_split.txt" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\address_split.txt" + } + for(def c : citys) { + index++ + def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city", "region", "name","address"]) + def rets = cut(dists) + FileUtils.batchCreateFile(rets, fileName, null, "utf-8", true); + } + } + + def cut(rets){ + ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + final CountDownLatch latch = new CountDownLatch(rets.size()) + def totalCount = new AtomicLong() + def rows = new CopyOnWriteArrayList() + for(def i=0;i Date: Thu, 27 Oct 2022 16:44:03 +0800 Subject: [PATCH 009/112] add logs --- .../com/fangjia/script/ccbevaluation/data/AddressCut.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy index e34c2b4af..e836327b2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy @@ -33,6 +33,7 @@ class AddressCut extends BaseBatch{ def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city", "region", "name","address"]) def rets = cut(dists) FileUtils.batchCreateFile(rets, fileName, null, "utf-8", true); + println index +" "+ c +" "+ rets.size() } } From 661871bee9f57fb577692cb2bc979760d24d342d Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 27 Oct 2022 16:47:28 +0800 Subject: [PATCH 010/112] add logs --- .../com/fangjia/script/ccbevaluation/data/AddressCut.groovy | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy index e836327b2..f8770132c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy @@ -61,12 +61,16 @@ class AddressCut extends BaseBatch{ street_number = jsonObj.result.streetNumber } if(street || street_number){ + + println totalCount.get() +" "+d.city +" "+d.address +" "+street+" "+street_number + rows << [d.city,d.region, d.name, d.address, street, street_number] // udata.put("m_date", new Date()) // DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) } } } + } catch (Exception e) { e.printStackTrace() } finally { From 8ee0ce68f4b0d5db4ad15cd7d5547365864d9e7c Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 28 Oct 2022 13:24:01 +0800 Subject: [PATCH 011/112] update --- .../BigdataCommunityBatch.groovy | 38 ++- .../ccbevaluation/BigdataDealBatch.groovy | 11 +- .../ccbevaluation/ExportOracleDataTest.groovy | 2 +- .../ccbevaluation/data/AddressCut.groovy | 37 +- .../ronghe/BigdataCitypriceBatch.groovy | 15 + .../ronghe/BigdataCommunityGJRHBatch.groovy | 104 +++--- .../fuzhou/DistrictPriceExportBatch.groovy | 107 +++--- .../fuzhou/DistrictSellStatBatch.groovy | 321 ++++++++++++++++++ .../fangjia/script/gj/ccb/CcbCodeUtils.groovy | 9 + .../gj/paimai/BeiKeCrawlByChrome.groovy | 10 +- .../gj/paimai/LianjiaCrawlByChrome.groovy | 8 +- .../thirdPriceCrawl/DataStandardBatch.groovy | 16 +- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 4 +- .../TestCrawlTotalCount.groovy | 4 +- 14 files changed, 549 insertions(+), 137 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index fafe61ba9..2439e06d8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -51,7 +51,7 @@ class BigdataCommunityBatch extends BaseBatch{ index ++ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"]) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"]) def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) @@ -175,25 +175,31 @@ class BigdataCommunityBatch extends BaseBatch{ ret << "pa101" ret << "" if(dist.address){ - def jsonObj = null - if(dist.address){ - jsonObj = FangjiaClientUtils.addressStructure(dist.city, dist.address) - } - if(jsonObj && jsonObj.result && jsonObj.result.street){ - if(jsonObj.result.street){ - ret << jsonObj.result.street - }else{ - ret << "" - } - - if(jsonObj.result.streetNumber){ - ret << jsonObj.result.streetNumber + if(dist.street){ + ret << dist.street + if(dist.street_number){ + ret << dist.street_number }else{ ret << "" } }else{ - ret << "" - ret << "" + def jsonObj = FangjiaClientUtils.addressStructure(dist.city, dist.address) + if(jsonObj && jsonObj.result && jsonObj.result.street){ + if(jsonObj.result.street){ + ret << jsonObj.result.street + }else{ + ret << "" + } + + if(jsonObj.result.streetNumber){ + ret << jsonObj.result.streetNumber + }else{ + ret << "" + } + }else{ + ret << "" + ret << "" + } } }else{ ret << "" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 789603d29..a1a81702d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -33,8 +33,8 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataDealBatch extends BaseBatch{ -// def startDate = 202206 -// def endDate = 202208 + def startDate = 202206 + def endDate = 202208 def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) @@ -58,11 +58,12 @@ class BigdataDealBatch extends BaseBatch{ public void process(Object args) { //def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date()) // def citys = CcbCommonUtils.citys; - def citys = ["上海"]//CcbCommonUtils.getRunningCitys(args) + def citys = CcbCommonUtils.getRunningCitys(args) if(args && args.size()>=2){ curDate = args[1].toString().toInteger() } + println "curDate:"+curDate // for(def city : citys){ //// if(!ccbCitys.contains(city)){ // addDealPriceMatch(city); @@ -110,9 +111,9 @@ class BigdataDealBatch extends BaseBatch{ def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(rets.size()) -// def lianjiaPriceMap = findThirdPartyPrice("上海") + def lianjiaPriceMap = findThirdPartyPrice("上海") def startDate = DateUtils.parseFromTightByPattern(DateUtils.addMonth(new Date(),-2),'yyyy-MM-dd') - def lianjiaPriceMap = getDistrictStatByCity("上海", startDate) +// def lianjiaPriceMap = getDistrictStatByCity("上海", startDate) for(def i=0;i + if(d['A']){ + def dbo = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district") + if(dbo){ + def udate = new BasicDBObject() + def street = null + def address = "" + if(d['E']){ + street = d['E'] + udate.put("street", street) + address = street + } + def streetNum = null + if(d['F']){ + streetNum = d['F'] + udate.put("street_number", streetNum) + address = address + streetNum + } + + if(dbo.address!=address){ + udate.put("address", address) + } + if(udate){ + udate.put("m_date", new Date()) + DBUtils.seaweed.update(new BasicDBObject('_id', dbo.get("_id")), new BasicDBObject('$set', udate)) + } + + } + } + + }) } def sameAddr() { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy new file mode 100644 index 000000000..1656648bd --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -0,0 +1,15 @@ +package com.fangjia.script.ccbevaluation.ronghe + +import com.fangjia.script.BaseBatch + +class BigdataCitypriceBatch extends BaseBatch{ + + static main(args) { + runBatch(new BigdataCitypriceBatch(), args) + } + + @Override + void process(Object args) { + + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy index b8116db22..77d788a46 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy @@ -53,62 +53,62 @@ class BigdataCommunityGJRHBatch extends BaseBatch { // }) // } - addBigdataDistrictFJW() +// addBigdataDistrictFJW() -// def citySeqnoMap = CcbCommonUtils.findDistrictSeqno(); + def citySeqnoMap = CcbCommonUtils.findDistrictSeqno(); + + def index = 0 + for (def city : citys) { + index++ + def districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city)) // -// def index = 0 -// for (def city : citys) { -// index++ -// def districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city)) -//// -//// for(int i=0;i<10;i++){ -//// if(districts.size()==0){ -//// Thread.sleep(1000*60) -//// districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city)) -//// }else{ -//// break -//// } -//// } -// -// def distMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["address","build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge"]) -// -// for(def dist : districts){ -// def obj = distMap.get(dist.commid)//CommonDatautils.findDistrictFieldById(dist.commid,["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"]) -// if(obj){ -// if(obj.alias && obj.alias.size()>0){ -// dist.commalias = obj.alias.get(0) -// }else{ -// dist.commalias = "" -// } -// dist.buildstru = CcbCodeUtils.getBuildStructureCode(obj.build_structure) -// dist.housetype = CcbCodeUtils.getDistHouseTypeCode(obj.estate_type2) -// dist.buildyear = CcbCodeUtils.getBuildYear(obj.finish_building_date); -// dist.houseuse = "11" -// dist.buildtype = CcbCodeUtils.parseBuildType(dist.build_type) -// dist.compid = new ObjectId() //物业公司id -// dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2) +// for(int i=0;i<10;i++){ +// if(districts.size()==0){ +// Thread.sleep(1000*60) +// districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city)) +// }else{ +// break // } // } -// println index +" "+city +" 小区加载完成 "+districts.size() -// addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city)) -// println index +" "+city +" 1.小区表完成" -// addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city)) -// println index +" "+city +" 2.小区建筑结构表" -// addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city)) -// println index +" "+city +" 3.小区房屋类型表" -// addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city)) -// println index +" "+city +" 4.小区建筑年代表" -// addBigdataCommfeeGj(districts, cityCodeDataMap.get(city)) -// println index +" "+city +" 5.小区费用表" -// addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city)) -// println index +" "+city +" 6.小区房屋用途表" -// addBigdataPositionGJ(districts, cityCodeDataMap.get(city), citySeqnoMap) -// println index +" "+ city + " 7.地理位置表" -// -//// addDealFieldInfo(distMap, cityCodeDataMap.get(city));//修改成交记录housestyle 一定要运行 -// -// } + + def distMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["address","build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge"]) + + for(def dist : districts){ + def obj = distMap.get(dist.commid)//CommonDatautils.findDistrictFieldById(dist.commid,["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"]) + if(obj){ + if(obj.alias && obj.alias.size()>0){ + dist.commalias = obj.alias.get(0) + }else{ + dist.commalias = "" + } + dist.buildstru = CcbCodeUtils.getBuildStructureCode(obj.build_structure) + dist.housetype = CcbCodeUtils.getDistHouseTypeCode(obj.estate_type2) + dist.buildyear = CcbCodeUtils.getBuildYear(obj.finish_building_date); + dist.houseuse = CcbCodeUtils.getUseTypeCode(obj.estate_type2)//"11" + dist.buildtype = CcbCodeUtils.parseBuildType(dist.build_type) + dist.compid = new ObjectId() //物业公司id + dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2) + } + } + println index +" "+city +" 小区加载完成 "+districts.size() + addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city)) + println index +" "+city +" 1.小区表完成" + addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city)) + println index +" "+city +" 2.小区建筑结构表" + addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city)) + println index +" "+city +" 3.小区房屋类型表" + addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city)) + println index +" "+city +" 4.小区建筑年代表" + addBigdataCommfeeGj(districts, cityCodeDataMap.get(city)) + println index +" "+city +" 5.小区费用表" + addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city)) + println index +" "+city +" 6.小区房屋用途表" + addBigdataPositionGJ(districts, cityCodeDataMap.get(city), citySeqnoMap) + println index +" "+ city + " 7.地理位置表" + +// addDealFieldInfo(distMap, cityCodeDataMap.get(city));//修改成交记录housestyle 一定要运行 + + } } def addBigdataCommunityGJRH(def districts, def cityid){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy index 517bfef5c..8d77112a6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy @@ -31,6 +31,8 @@ class DistrictPriceExportBatch extends BaseBatch{ public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com") public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName("business", "fz_district", "mongo.db.fangjia.com") + static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com") + def specialDistsPriceMap = ["津泰新村7号、9-17号":53620,"津泰新村14号":76230,"津泰新村1-6号、8号":32580] static main(args) { runBatch(new DistrictPriceExportBatch(), args) @@ -41,18 +43,8 @@ class DistrictPriceExportBatch extends BaseBatch{ def headers = ["id","comm_id","city","district","name","avg_price","year_month"] - def citys = CommonDatautils.findAllCity() - citys.remove("福州"); - citys.add(0, "福州") - - def fz_datas = [] - for(def city : citys){ - def ret = createDatas("福州") - saveToDB(ret) - if(city == "福州"){ - fz_datas = ret - } - } + def ret = createNewDatas("福州") + saveToDB(ret) def fileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".csv" def dataCodeFileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".sha512" @@ -60,7 +52,7 @@ class DistrictPriceExportBatch extends BaseBatch{ fileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".csv" dataCodeFileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".sha512" } - def sb = appendResult(headers, fz_datas) + def sb = appendResult(headers, ret) DataHandleUtils.outPutFile(fileName, sb, charset); FzFtpUtils.ftpUpload(fileName) //生成加密数据文件并上传 @@ -106,16 +98,50 @@ class DistrictPriceExportBatch extends BaseBatch{ return rets } + + def createNewDatas(def city){ + def districts = getFzDistricts(city) + def curPriceMap = findFzDistrictSellStat("福州",curDate) + def result = [] + for(def rMap : districts){ + def curPrice = 0 + def dbo = null + if(curPriceMap.containsKey(rMap.get("_id").toString())){ + dbo = curPriceMap.get(rMap.get("_id").toString()) + curPrice = dbo.avg_price + }else{ + curPrice = specialDistsPriceMap.get(rMap.name) + if(curPrice){ + def randMarkup = nextDouble(0, 1) + curPrice = (int)(curPrice*randMarkup/100 + curPrice) + } + } + def m = [] + if(curPrice && dbo){ + m << dbo.get("_id").toString()//均价ID + }else{ + m << new ObjectId().toString() + } + m << rMap.get("_id").toString()//楼盘ID + m << rMap.city + m << rMap.region + m << rMap.name + if(curPrice){ + m << curPrice + }else{ + m << 0 + } + m << curDate.toInteger(); + result << m + } + return result + } + def createDatas(def city){ // def districts = CommonDatautils.findAllDistricts(city, "district", null); // def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01"); // districts.addAll(removeDists) - def districts = [] - if(city == "福州"){ - districts = getFzDistricts(city) - }else{ - districts = CommonDatautils.findAllDistricts(city, "district", null); - } + def districts = getFzDistricts(city) def endDateStr = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-10" def dataStr = DateUtils.add(endDateStr, -30) @@ -128,38 +154,27 @@ class DistrictPriceExportBatch extends BaseBatch{ // def key = row[2] + "##" + row[3] + "##" + row[4] // map2.put(key, row[5]) // } - def lastMonthPriceMap = findFzDistrictStat("福州",lastMonthDate) + def result = [] + for(def rMap : districts){ - def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate) + def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, curDate) def curPrice = 0 if(dbo){ def lastMonthPrice = lastMonthPriceMap.get(rMap.get("_id").toString()) curPrice = dbo.get("avg_price") if(lastMonthPrice && curPrice && dbo.value_type!=1){ def markup = (curPrice - lastMonthPrice)*100/lastMonthPrice - if(city == "福州"){ - if(Math.abs(markup)>3){ - def randMarkup = nextDouble(0, 3) - if(markup < 0){ - markup = randMarkup*(-1) - }else{ - markup = randMarkup - } - curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice) - } - }else{ - if(Math.abs(markup)>10){ - def randMarkup = nextDouble(0, 10) - if(markup < 0){ - markup = randMarkup*(-10) - }else{ - markup = randMarkup - } - curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice) + if(Math.abs(markup)>3){ + def randMarkup = nextDouble(0, 3) + if(markup < 0){ + markup = randMarkup*(-1) + }else{ + markup = randMarkup } + curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice) } } }else{ @@ -190,6 +205,17 @@ class DistrictPriceExportBatch extends BaseBatch{ return result } + def findFzDistrictSellStat(city,lastMonthDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("date", lastMonthDate) + def priceMap = [:] + DISTRICT_SELL_STAT.find(q).each{ + priceMap.put(it.commid, it) + } + return priceMap + } + def findFzDistrictStat(city,lastMonthDate){ def q = new BasicDBObject() @@ -256,7 +282,6 @@ class DistrictPriceExportBatch extends BaseBatch{ udata.put("avg_price", r[5]) FZ_DISTRICT_STAT_COLL.update(q, new BasicDBObject('$set', udata)) }else{ - def udata = new BasicDBObject() def o = new BasicDBObject() println r[0] o.put("_id", new ObjectId(r[0])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy new file mode 100644 index 000000000..a11d0ce17 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy @@ -0,0 +1,321 @@ +package com.fangjia.script.fuzhou + +import com.fangjia.core.util.DataUtil +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.QueryOperators + +import java.text.DecimalFormat +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +@BatchOwner(Watcher.GaoJuan) +class DistrictSellStatBatch extends BaseBatch { + + static DBCollection FZ_DISTRICT_STAT = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com") + + static DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") + static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com") + static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName("business", "fz_district", "mongo.db.fangjia.com") + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + + + public static void main(String[] args) { + runBatch(new DistrictSellStatBatch(), args); + } + + @Override + void process(Object args) { + calcMonthlyPrice() + } + + + def getMomDate(int date){ + + Calendar calendar = Calendar.getInstance() + calendar.setTime(Date.parse("yyyyMM", String.valueOf(date))) + calendar.add(Calendar.MONTH, -1) + return calendar.getTime().format("yyyyMM").toInteger() + } + + def getYoyDate(int date){ + + Calendar calendar = Calendar.getInstance() + calendar.setTime(Date.parse("yyyyMM", String.valueOf(date))) + calendar.add(Calendar.YEAR, -1) + return calendar.getTime().format("yyyyMM").toInteger() + } + + def calcMonthlyPrice(){ + def citys =["鞍山", '大庆', "衡阳", "南阳", "莆田"] // CommonUtils.findAllCity() + def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1) + + def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(new Date())) + + def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def monDate = getMomDate(curMonth) + def yoyDate = getYoyDate(curMonth) + + def index = 0 + for(def city : citys){ + + index++ + def districts = [] + if(city == "福州"){ + districts = FZ_DIST_COLL.find(new BasicDBObject("city", city), keys).toList() + }else{ + districts = DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000) + } + + def weekendPriceMap = findPriceMap(city, lastMonthWeekend)//上个月小区均价 + def curMonthlyPriceMap = findMonthlyPrice(city, curMonth)//当月小区月均价 + def lastMonthlyPriceMap = findMonthlyPrice(city, monDate)//上个月小区月均价 + def lastYearPriceMap = findMonthlyPrice(city, yoyDate)//去年小区月均价 + if(districts && weekendPriceMap){ + running(curMonth, districts, weekendPriceMap, curMonthlyPriceMap, lastMonthlyPriceMap, lastYearPriceMap) + } + println index+" "+city +" "+ (new Date()).toString() + } + } + + def running(districts){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + districts.each{ dist-> + fixedThreadPool.execute(new Runnable() { + void run() { + try { + totalCount.addAndGet(1) + println totalCount.get()+" "+ dist.city+" "+dist.name + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + } + + def running(curMonth, districts, weekendPriceMap, curMonthlyPriceMap, lastMonthlyPriceMap, lastYearPriceMap){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + def rets = new CopyOnWriteArrayList() + + districts.each{dist-> + fixedThreadPool.execute(new Runnable() { + void run() { + try { + totalCount.addAndGet(1) + def key = dist.region +"##" + dist.name + def weekendPriceObj = weekendPriceMap.get(key) + if(weekendPriceObj){ + def curPriceObj = curMonthlyPriceMap.get(key) + def lastMonthPriceObj = lastMonthlyPriceMap.get(key) + def lastYearPriceObj = lastYearPriceMap.get(key) + double mom = 0d + + if(lastMonthPriceObj) { + mom = DataUtil.roundDouble(((weekendPriceObj.avg_price - lastMonthPriceObj.avg_price)/(double)lastMonthPriceObj.avg_price),4) + } + + def curPrice = null; + if(curPriceObj){ + curPrice = curPriceObj.avg_price + }else{ + curPrice = weekendPriceObj.avg_price + } + + if(weekendPriceObj.value_type!=1){ + if(dist.city == "福州"){ + if(Math.abs(mom)*100>3){ + def randMarkup = nextDouble(0, 3) + if(mom < 0){ + mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4) + }else{ + mom = DataUtil.roundDouble(randMarkup/100, 4) + } + curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price) + } + }else{ + if(Math.abs(mom)*100>10){ + def randMarkup = nextDouble(0, 10) + if(mom < 0){ + mom = DataUtil.roundDouble(randMarkup*(-1)/100,4) + }else{ + mom = DataUtil.roundDouble(randMarkup/100,4) + } + curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price) + } + } + } + + double yoy = 0d + if(lastYearPriceObj){ + yoy = DataUtil.roundDouble(((curPrice - lastYearPriceObj.avg_price)/(double)lastYearPriceObj.avg_price),4) + } + if(curPriceObj){ + if(curPrice!=weekendPriceObj.avg_price){ + def data = new BasicDBObject() + data.put("m_date", new Date()) + data.put("commid", dist._id.toString()) + data.put("mom", mom) + data.put("yoy", yoy) + data.put("avg_price", curPrice) + DISTRICT_SELL_STAT.update(new BasicDBObject("_id", curPriceObj._id) , new BasicDBObject('$set', data),true,false) + println "[update]"+totalCount.get()+" "+ dist.city+" "+dist.name + } + }else{ + def data = new BasicDBObject() + data.put("city", dist.city) + data.put("region", dist.region) + data.put("district_name", dist.name) + data.put("estate_type", dist.estate_type2) + data.put("date", curMonth) + data.put("avg_price", curPrice) + data.put("credit", weekendPriceObj.credit) + data.put("m_date", new Date()) + data.put("commid", dist._id.toString()) + data.put("mom", mom) + data.put("yoy", yoy) + DISTRICT_SELL_STAT.save(data) + println "[save]"+totalCount.get()+" "+ dist.city+" "+dist.name + } + } + + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + }; + }) + } + latch.await() + } + + + //随机浮点数 + public static double nextDouble(final double min, final double max) throws Exception { + + DecimalFormat df = new DecimalFormat("#.00"); + if (max < min) { + throw new Exception("min < max"); + } + if (min == max) { + return min; + } + return Double.parseDouble(df.format(min + ((max - min) * new Random().nextDouble()))); + } + + + def findPriceMap(city, weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("weekend", weekend) + def dbcursor = DISTRICT_STAT.find(q) + def priceMap = [:] + for(def d : dbcursor){ + priceMap.put(d.region +"##"+d.district_name, d) + } + return priceMap + } + + def findMonthlyPrice(city, monDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("date", monDate) + def dbcursor = DISTRICT_SELL_STAT.find(q) + def priceMap = [:] + for(def d : dbcursor){ + priceMap.put(d.region +"##"+d.district_name, d) + } + return priceMap + } + + + +// def updateDistrictSellStat(citys){ +// def index = 0 +// citys.each{city-> +// index ++ +// def stats = FZ_DISTRICT_STAT.find(new BasicDBObject("month",202209).append("city", city).append("avg_price", new BasicDBObject(QueryOperators.GTE, 0))).asList() +// if(stats){ +// batchUpdate(stats) +// } +// println "[${index}] "+city+" 完成" +// } +// } +// +// def batchUpdate(def stats){ +// def totalCount = new AtomicLong() +// final CountDownLatch latch = new CountDownLatch(stats.size()) +// +// for(def i=0;i -// fixedThreadPool.execute( -// new Runnable() { -// void run() { + fixedThreadPool.execute( + new Runnable() { + void run() { try{ println "(${rowIndex}-${count}-${count_.incrementAndGet()})【${dbo.city}-${dbo.region}-${dbo.name}-${dbo.address}-${dbo.source}】" if(dbo.name){ @@ -162,12 +162,12 @@ class DataStandardBatch extends BaseBatch{ }finally { latch.countDown() } -// } -// } -// ); + } + } + ); } -// latch.await() + latch.await() long end = System.currentTimeMillis() println "第【${rowIndex}】次中匹配成功的:"+successCount+" 个,耗时:"+ (end - start)+" "+ DateUtils.format(new Date(),"yyyy-MM-dd HH:mm:ss") @@ -920,7 +920,7 @@ class DataStandardBatch extends BaseBatch{ def findAllNoMatchData(){ DBObject qdata = new BasicDBObject(); qdata.put("s_date", SHORT_DATE) -// qdata.put("city","北京") + qdata.put("city","福州") // qdata.put("source","安居客") // qdata.put("name",Pattern.compile("龚家宅.*")) qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false)); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index f954743e4..1caff80e1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -25,7 +25,7 @@ class SynchDistrictPrice extends BaseBatch{ def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"] // "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产" - def esSources = ["中原地产":"中原地产"] + def esSources = [ "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产"] static main(args) { @@ -41,7 +41,7 @@ class SynchDistrictPrice extends BaseBatch{ println "starting......" def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd") - def citys = CommonUtils.findAllCity() + def citys = ["福州"]//CommonUtils.findAllCity() esSources.each{esSource, mgSource-> for(def city : citys){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index 99355ab2a..22eedff7f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] for(def source :sources){ - def count1 = findTotalCount(source, 202208); - def count2 = findTotalCount(source, 202209); + def count1 = findTotalCount(source, 202209); + def count2 = findTotalCount(source, 202210); println source +" "+count1 +" "+count2 } } From 61b7ac8fca3518e96936bd88d24dcdf70f04ec9d Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 28 Oct 2022 15:58:05 +0800 Subject: [PATCH 012/112] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=BB=BA=E8=A1=8C86?= =?UTF-8?q?=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/ExportOracleDataTest.groovy | 1171 +++++++++++++++++ .../ccbevaluation/data/AddressCut.groovy | 61 + .../ccbevaluation/ronghe/SSHHelperTest.groovy | 112 ++ 3 files changed, 1344 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/SSHHelperTest.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy new file mode 100644 index 000000000..a93ceb174 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy @@ -0,0 +1,1171 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.core.RegionData +import com.fangjia.script.ccbevaluation.ronghe.SSHHelper +import com.fangjia.script.ccbevaluation.ronghe.SSHHelperTest +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.jcraft.jsch.ChannelSftp +import groovy.sql.Sql +import org.apache.commons.codec.digest.DigestUtils + +import java.sql.ResultSet + +class ExportOracleDataTest { + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + def citys = null + + static main(args) { + def task = new ExportOracleDataTest() + task.run(args) + } + + def run(args) { + citys = CcbCommonUtils.getRunningCitys(args) + def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); + def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") + def new_dists = [] + //不会变 10.25 + exportEvaluationRateData() + exportBigdataPoiData()// + exportBigdataDistrictData()// + + exportGuidedPrice()//随人工改变// + exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策// + + //处理小区情况 + exportBigdataCommunity()// + exportCommunityStaging()//没cityid + exportCommunityBuileType()// + + exportThirdPartyPrice2(lastMonth, new_dists)// + exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 + exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 + + //第三方网站权重表 + exportWebsiteRate(); + + //计算结果相关 + exportAvmEvaluationGJNB(curMonth)//无城市区分 + exportAVMEvaluationItemGJNB(curMonth)//无城市区分 + + exportCityPrice(curMonth)//无城市区分 + exportRegionPrice(curMonth)//无城市区分 + + exportCommunityBasePriceGj(curMonth)//已合下面的合并 //无城市区分 + exportCommunityBasePrice(curMonth)//无城市区分 + + exportCommunityPdescribeGj(curMonth)//无城市区分 + exportCommCaseDetailGj(curMonth)//无城市区分 + + def curDate = curMonth.toString()+"28" + // ok文件 +// exportInOk() + SSHHelperTest jschCommand = new SSHHelperTest(); + System.out.println(jschCommand.executeCommand("sh /home/ccb/output/eval_package.sh ${curDate}")); + + } + + + def exportEvaluationRateData() { + def sql = "select ID,RATE_TYPE,RATE,RANK,CITY,REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from evaluate_rate_gj"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.RATE_TYPE + data << r.RATE + data << r.RANK + data << r.CITY + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["系数ID","系数类型","系数","排序","城市","备注","创建时间","更新时间","来源公司"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "上传Ftp完成......" + } + + + def exportWebsiteRate(){ + def sql = "select DATA_SOURCE_ID,DATA_SOURCE,DATA_SOURCE_MAP,WEIGHT,TYPE,SOURCECODE,CITYID,CITYNAME,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from WEBSITE_CONFIG_GJ"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.DATA_SOURCE_ID + data << r.DATA_SOURCE + data << r.DATA_SOURCE_MAP + data << r.WEIGHT + data << r.TYPE + data << r.SOURCECODE + data << r.CITYID + data << r.CITYNAME + data << r.CREATETIME + data << r.UPDATETIME + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["系数ID","系数类型","系数","排序","城市","备注","创建时间","更新时间","来源公司"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "上传Ftp完成......" + } + + + def exportBigdataPoiData() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID, DISTID, ID, POI_NAME, TYPE, TYPE_CODE, LATITUDE, LONGITUDE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from bigdata_poi_gj where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.CITYID + data << r.DISTID + data << r.ID + data << r.POI_NAME + data << r.TYPE + data << r.TYPE_CODE + data << r.LATITUDE + data << r.LONGITUDE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","来源公司"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "BIGDATA_POI_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "BIGDATA_POI_GJ 上传Ftp完成......" + } + + def exportCommunityStaging() { + + def allRows = [] + def sql = "select ID,COMMID,MASTER_COMMID,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_staging_gj"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMID + data << r.MASTER_COMMID + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["分期ID","小区ID","小区主期ID","备注","创建时间","更新时间","来源公司"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "COMMUNITY_STAGING_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "COMMUNITY_STAGING_GJ 上传Ftp完成......" + } + + def exportCommunityBuileType() { +// def citys = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"] +// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID, COMMID, BUILDTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, WORKDATE from bigdata_commbuildtype_gj where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.BUILDTYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.WORKDATE + allRows << data + }); + } + + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "BIGDATA_COMMBUILDTYPE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "BIGDATA_COMMBUILDTYPE_GJ 上传Ftp完成......" + } + + def exportBigdataDistrictData() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select DISTID, DISTNAME, DISTSPELL, CITYID, DISTCODE, SEQ, DISTTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE, LJXZQH, LJXZQHDM, XZQHDM, WORKDATE from bigdata_district_gj where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.DISTID + data << r.DISTNAME + data << r.DISTSPELL + data << r.CITYID + data << r.DISTCODE + data << r.SEQ + data << r.DISTTYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.LJXZQH + data << r.LJXZQHDM + data << r.XZQHDM + data << r.WORKDATE + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" + } + def headers = [] +//["行政区ID","行政区名称","行政区名称拼音","城市ID","国标码","显示顺序","行政区类别","备注","创建时间","更新时间","来源公司","逻辑行政区","逻辑行政区划代码","行政区划代码","数据时间"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "BIGDATA_DISTRICT_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "BIGDATA_DISTRICT_GJ 上传Ftp完成......" + } + + + def exportGuidedPrice() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE,AVG_PRICE, GUIDING_PERSON, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE from guided_price_gj where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMID + data << r.COMMNAME + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.DEALDATE + data << r.AVG_PRICE + data << r.GUIDING_PERSON + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" + } + def headers = [] +//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "GUIDED_PRICE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "GUIDED_PRICE_GJ 上传Ftp完成......" + } + + def exportCommunityGuidedPrice(def date) { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where CITYID='${cityCode}' and createtime>=to_date('${date}','yyyyMMdd')"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMID + data << r.COMMNAME + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.AVGPRICE + data << r.SOURCETYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导价格","类型","备注","创建时间","更新时间","来源公司"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "COMMUNITY_GUIDE_PRICE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "COMMUNITY_GUIDE_PRICE_GJ 上传Ftp完成......" + } + + def exportBigdataDeal(def dealDate, def new_dists) { + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + + "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + + "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + + "from bigdata_deal_gj where CITYID='${cityCode}' and dealdate>=to_date('${dealDate}','yyyy-MM-dd') and distid not in ${newstr}" + def rets1 = dealData(sql1) + + def sql2 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + + "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + + "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + + "from bigdata_deal_gj where CITYID='${cityCode}' and distid in ${newstr}" + def rets2 = dealData(sql2) + allRows.addAll(rets1) + allRows.addAll(rets2) + }else{ + def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + + "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + + "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + + "from bigdata_deal_gj where CITYID='${cityCode}' and dealdate>=to_date('${dealDate}','yyyy-MM-dd')"; + allRows.addAll(dealData(sql)) + } + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "BIGDATA_DEAL_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "BIGDATA_DEAL_GJ 上传Ftp完成......" + } + + + def dealData(def sql){ + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.CITYID + data << r.DEALID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.STREET + data << r.STREETNO + data << r.COMMID + data << r.COMMNAME + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.UNITID + data << r.UNITNO + data << r.UNITNAME + data << r.HOUSEID + data << r.ADDRESS + data << r.BUILDYEAR + data << r.PROPTYPE + data << r.PROPYEARS + data << r.PERIODNUM + data << r.FLOORNUM + data << r.FLOORTYPE + data << r.FLOORCNT + data << r.BUILDTYPE + data << r.BUILDSTRU + data << r.BUILDAREA + data << r.LIVINGAREA + data << r.HOUSESTYLE + data << r.ROOMCNT + data << r.LIVINGROOMCNT + data << r.KITCHENCNT + data << r.BATHROOMCNT + data << r.HOUSEUSE + data << r.HOUSESTRU + data << r.HOUSEASPECT + data << r.FITMENT + data << r.EVALUATENO + data << r.DEALUNITPRICE + data << r.DEALTOTALPRICE + data << r.DECLAREPRICE + data << r.EVALUATEPRICE + data << r.CONTRACTNO + data << r.DEALDATE + data << r.DEALTYPE + data << r.PAYTYPE + data << r.DEALPERIOD + data << r.IFELEVATOR + data << r.COMPAREAGARDEN + data << r.COMPAREATERRACE + data << r.COMPAREABALCONY + data << r.COMPAREABAYWINDOW + data << r.COMPAREAATTIC + data << r.COMPAREAGARAGE + data << r.COMPAREABASEMENT + data << r.COMPAREAOTHER + data << r.DATASOURCE + data << r.AGENCYID + data << r.AGENCYNAME + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.CJJE + data << r.WORKDATE + allRows << data + }); + return allRows + } + + def exportThirdPartyPrice() { + def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, ZHUGE_AVG_PRICE, ANJUKE_AVG_PRICE, FANG_AVG_PRICE, LIANJIA_AVG_PRICE, CITYHOUSE_AVG_PRICE, MAITIAN_AVG_PRICE, KE_AVG_PRICE, \n" + + "TAOFANG_AVG_PRICE, ZHONGYUAN_AVG_PRICE, TBS_AVG_PRICE, QFANG_AVG_PRICE, FANGTAN_AVG_PRICE, FANGDUODUO_AVG_PRICE, WIWJ_AVG_PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE\n" + + "from third_party_price_gj"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMID + data << r.COMMNAME + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.YEAR_MONTH + data << r.ZHUGE_AVG_PRICE + data << r.ANJUKE_AVG_PRICE + data << r.FANG_AVG_PRICE + data << r.LIANJIA_AVG_PRICE + data << r.CITYHOUSE_AVG_PRICE + data << r.MAITIAN_AVG_PRICE + data << r.KE_AVG_PRICE + data << r.TAOFANG_AVG_PRICE + data << r.ZHONGYUAN_AVG_PRICE + data << r.TBS_AVG_PRICE + data << r.QFANG_AVG_PRICE + data << r.FANGTAN_AVG_PRICE + data << r.FANGDUODUO_AVG_PRICE + data << r.WIWJ_AVG_PRICE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE.replaceAll(~'\\r', '') + allRows << data + }) + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] +//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "THIRD_PARTY_PRICE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "THIRD_PARTY_PRICE_GJ 上传数据文件完成......" + } + + + def exportThirdPartyPrice2(price_date, new_dists) { + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" +// fileName = "D:\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] + def index = 0 + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def allRows = [] + if(new_dists) { + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " + + "COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " + + "CITYID='${cityCode}' and price_date=${price_date} and distid not in ${newstr} "; + def rets1 = getThirdPartyDatas(sql1) + allRows.addAll(rets1) + + def sql2 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " + + "COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " + + "CITYID='${cityCode}' and distid in ${newstr} "; + def rets2 = getThirdPartyDatas(sql2) + allRows.addAll(rets2) + }else{ + def sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where CITYID='${cityCode}' and price_date=${price_date} "; + allRows = getThirdPartyDatas(sql) + } + if(allRows && allRows.size()>0){ + index = index + 1 + if(index==1){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + } + } + } + + println "BIGDATA_THIRD_PARTY_PRICE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + } + + def getThirdPartyDatas(def sql){ + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.CITYID + data << r.DISTID + data << r.COMMID + data << r.COMMNAME + data << r.DATA_SOURCE_ID + data << r.DATA_SOURCE + data << r.PRICE_DATE + data << r.COMMUNITY_PRICE + data << r.COMMUNITY_MAX_PRICE + data << r.COMMUNITY_MIN_PRICE + data << r.UPDATETIME + allRows << data + }) + return allRows; + } + + def exportBigdataCommunity(){ + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID, DISTID, COMMID, DISTNAME, COMMNAME, ADDRESS, LATITUDE, LONGITUDE, LANDAREA, BUILDAREA, GREENINGRATE, PLOTRATIO, PARKINGCNT, BUILDYEAR, HOUSEUSE, ZTS, WORKDATE, UPDATEFLAG, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE, \n" + + "\tCOMMALIAS, STREET, STREETNO, IFNEW, INTRODUCTION, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, SOURCECODE, XZJDB, LJXZQHDM, LJXZQH,STATUS from bigdata_community_gj where CITYID='${cityCode}' ";//where cityId in (350200,150100) + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.DISTID + data << r.COMMID + data << r.DISTNAME + data << r.COMMNAME + data << r.ADDRESS + data << r.LATITUDE + data << r.LONGITUDE + data << r.LANDAREA + data << r.BUILDAREA + data << r.GREENINGRATE + data << r.PLOTRATIO + data << r.PARKINGCNT + data << r.BUILDYEAR + data << r.HOUSEUSE + data << r.ZTS + data << r.WORKDATE + data << r.UPDATEFLAG + data << r.UPDATETIME + data << r.COMMTYPE + data << r.COMMALIAS + data << r.STREET + data << r.STREETNO + data << r.IFNEW + data << "" + data << r.REMARK + data << r.CREATETIME + data << r.SOURCECODE + data << r.XZJDB + data << r.LJXZQHDM + data << r.LJXZQH + data << r.STATUS + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途", + //"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司", + //"乡镇/街道办","逻辑行政区划代码","逻辑行政区划","状态"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "BIGDATA_COMMUNITY_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "BIGDATA_COMMUNITY_GJ 上传Ftp完成......" + } + + def exportBigdataForsale(publishDate, new_dists){ + //["北京","上海"] + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"] + def index = 0 + for(def city : citys){ + def cityCode = cityCodeMap.get(city) + + def allRows = [] + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj " + + "where cityId='${cityCode}' and publishtime>=to_date('${publishDate}','yyyy-MM-dd') and distid not in ${newstr}"; + def rets1 = getHouseDatas(sql1) + allRows.addAll(rets1) + }else{ + def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}' and publishtime>=to_date('${publishDate}','yyyy-MM-dd')"; + allRows = getHouseDatas(sql) + } + if(allRows.size()>0){ + index = index + 1 + if(index == 1){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + } + + } + } + + if(new_dists){//补充被修区域的数据 + def newstr = "('"+new_dists.join("','")+"')" + def sql2 = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH " + + "from bigdata_forsale_gj where distid in ${newstr}" + def rets2 = getHouseDatas(sql2) + if(rets2.size()>0){ + FileUtils.batchCreateFile(rets2, fileName, headers, "utf-8", true); + } + } + println "BIGDATA_FORSALE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "BIGDATA_FORSALE_GJ 上传Ftp完成......" + } + + def getHouseDatas(sql){ + def allRows = [] + def index = 0 + sqlInstance.eachRow(sql,{r-> + index = index+1 + println r.CITYNAME +" "+index + def data = [] + data << r.CITYID + data << r.FORSALEID + data << r.PROPCODE + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.UNITID + data << r.UNITNO + data << r.UNITNAME + data << r.HOUSEID + data << r.ADDRESS + data << r.BUILDYEAR + data << r.PROPTYPE + data << r.PROPYEARS + data << r.UNITPRICE + data << r.TOTALPRICE + data << r.PAYTYPE + data << r.FLOORNUM + data << r.FLOORCNT + data << r.BUILDTYPE + data << r.HOUSESTRU + data << r.BUILDSTRU + data << r.BUILDAREA + data << r.LIVINGAREA + data << r.HOUSESTYLE + data << r.ROOMCNT + data << r.LIVINGROOMCNT + data << r.KITCHENCNT + data << r.BATHROOMCNT + data << r.HOUSEASPECT + data << r.FITMENT + data << r.IFELEVATOR + data << r.ELEVATORHOUSE + data << r.PUBLISHTIME + data << r.EXPIRETIME + data << r.IFFIRST + data << r.DATASOURCE + data << r.AGENCYID + data << r.AGENCYNAME + data << r.PUBLISHER + data << r.PHONE + data << "" + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.ATTACH + allRows << data +// if(allRows.size()%200000==0){ +// FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); +// allRows.clear() +// } + }); + return allRows + } + + + def exportAvmEvaluationGJNB(yearMonth){ + def sql = "select ID,CITYID,CITYNAME, YEAR_MONTH, COMMUNITY_NUM, HAS_PRICE_COMMUNITY_NUM, PRICE_COVERAGE, ARTIFICIAL_CALC_NUM, ARTIFICIAL_CALC_COVERAGE, SAMPLE_CALC_NUM, SAMPLE_CALC_COVERAGE, \n" + + "ANN_CALC_NUM, ANN_CALC_COVERAGE, TEST_NUM, TEST_COMM_NUM, TEST_COMM_COVERAGE, PPE15_COVERAGE, RATE_MEDIAN, COEFFICIENT,DEAL_TEST_CASE_NUM, DEAL_TEST_COMM_NUM, DEAL_TEST_COMM_COVERAGE, DEAL_PPE15_COVERAGE, \n" + + "DEAL_COEFFICIENT, DEAL_RATE_MEDIAN, THIRD_TEST_CASE_NUM, THIRD_TEST_COMM_NUM, THIRD_TEST_COMM_COVERAGE, THIRD_PPE15_COVERAGE, THIRD_COEFFICIENT, THIRD_RATE_MEDIAN, \n" + + "to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from avm_evaluation_gj where YEAR_MONTH = ${yearMonth}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.YEAR_MONTH + data << r.COMMUNITY_NUM + data << r.HAS_PRICE_COMMUNITY_NUM + data << r.PRICE_COVERAGE + data << r.ARTIFICIAL_CALC_NUM + data << r.ARTIFICIAL_CALC_COVERAGE + data << r.SAMPLE_CALC_NUM + data << r.SAMPLE_CALC_COVERAGE + data << r.ANN_CALC_NUM + data << r.ANN_CALC_COVERAGE + data << r.TEST_NUM + data << r.TEST_COMM_NUM + data << r.TEST_COMM_COVERAGE + data << r.PPE15_COVERAGE + data << r.RATE_MEDIAN + data << r.COEFFICIENT + data << r.DEAL_TEST_CASE_NUM + data << r.DEAL_TEST_COMM_NUM + data << r.DEAL_TEST_COMM_COVERAGE + data << r.DEAL_PPE15_COVERAGE + data << r.DEAL_COEFFICIENT + data << r.DEAL_RATE_MEDIAN + data << r.THIRD_TEST_CASE_NUM + data << r.THIRD_TEST_COMM_NUM + data << r.THIRD_TEST_COMM_COVERAGE + data << r.THIRD_PPE15_COVERAGE + data << r.THIRD_COEFFICIENT + data << r.THIRD_RATE_MEDIAN + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" + fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","YEAR_MONTH","COMMUNITY_NUM","HAS_PRICE_COMMUNITY_NUM","PRICE_COVERAGE","ARTIFICIAL_CALC_NUM","ARTIFICIAL_CALC_COVERAGE","SAMPLE_CALC_NUM","SAMPLE_CALC_COVERAGE", + //"ANN_CALC_NUM","ANN_CALC_COVERAGE","TEST_NUM","TEST_COMM_NUM","TEST_COMM_COVERAGE","PPE15_COVERAGE","RATE_MEDIAN","COEFFICIENT","DEAL_TEST_CASE_NUM","DEAL_TEST_COMM_NUM","DEAL_TEST_COMM_COVERAGE","DEAL_PPE15_COVERAGE", + //"DEAL_COEFFICIENT","DEAL_RATE_MEDIAN","THIRD_TEST_CASE_NUM","THIRD_TEST_COMM_NUM","THIRD_TEST_COMM_COVERAGE","THIRD_PPE15_COVERAGE","THIRD_COEFFICIENT","THIRD_RATE_MEDIAN","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false); + println "AVM_EVALUATION_GJNB 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + CcbFtpTestUtils.ftpUpload(null, fileName1, curDate) + println "AVM_EVALUATION_GJNB 上传Ftp完成......" + } + + def exportAVMEvaluationItemGJNB(yearMonth){ + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, CONFIDENCE,TARGET_AVG_PRICE, \n" + + "to_char(DEALDATE,'yyyy-mm-dd hh24:mi:ss') as DEALDATE, UNIT_PRICE, BUILDAREA, FITMENT, HOUSEASPECT, FLOORNUM, FLOORCNT, DATA_TYPE, RATE, PE, \n" + + "to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME from avm_evaluation_item_gj where YEAR_MONTH = ${yearMonth}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.AVG_PRICE + data << r.CALC_TYPE + data << r.CONFIDENCE + data << r.TARGET_AVG_PRICE + data << r.DEALDATE + data << r.UNIT_PRICE + data << r.BUILDAREA + data << r.FITMENT + data << r.HOUSEASPECT + data << r.FLOORNUM + data << r.FLOORCNT + data << r.DATA_TYPE + data << r.RATE + data << r.PE + data << r.CREATETIME + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" + fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","CONFIDENCE","TARGET_AVG_PRICE","DEALDATE","UNIT_PRICE","BUILDAREA","FITMENT","HOUSEASPECT","FLOORNUM","FLOORCNT","DATA_TYPE","RATE","PE","CREATETIME"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false); + println "AVM_EVALUATION_ITEM_GJNB 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + CcbFtpTestUtils.ftpUpload(null, fileName1, curDate) + println "AVM_EVALUATION_ITEM_GJNB 上传Ftp完成......" + } + + def exportCommunityBasePrice(yearMonth){ + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.AVG_PRICE + data << r.CALC_TYPE + data << r.FAKE_RATE + data << r.DIFF_RATE + data << r.CONFI_RATE + data << r.CORR_RATE + data << r.CONFIDENCE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false); + println "COMMUNITY_BASE_PRICE_GJNB 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + CcbFtpTestUtils.ftpUpload(null, fileName1, curDate) + println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......" + } + + def exportCommunityBasePriceGj(yearMonth){ + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.AVG_PRICE + data << r.CALC_TYPE + data << r.FAKE_RATE + data << r.DIFF_RATE + data << r.CONFI_RATE + data << r.CORR_RATE + data << r.CONFIDENCE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "COMMUNITY_BASE_PRICE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "COMMUNITY_BASE_PRICE_GJ 上传Ftp完成......" + } + + def exportCityPrice(month){ + def sql = "select ID, CITYID, CITYNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from city_price_gj where year_month=${month}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.PRICE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def fileName = "/ccb/MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "CITY_PRICE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "CITY_PRICE_GJ 上传Ftp完成......" + } + + def exportRegionPrice(month){ + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from DISTRICT_PRICE_GJ where year_month=${month}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.PRICE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def fileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "DISTRICT_PRICE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "DISTRICT_PRICE_GJ 上传Ftp完成......" + } + + def exportCommunityPdescribeGj(month){ + +// def citys = CcbCommonUtils.citys; +// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] +// +// for(def city : citys) { +// def cityCode = cityCodeDataMap.get(city) + def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month}"; +// def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time in (202110,202111,202112,202201)"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMUNITY_ID + data << r.CITYID + data << r.CITY_NAME + data << r.DISTRICT_CODE + data << r.DISTRICT_NAME + data << r.COMMUNITY_NAME + data << r.PQ_NAME + data << r.BASE_PRICE + data << r.CONFIDENCE_LEVEL + data << r.CNT + data << r.MIN_PRICE + data << r.MEDIANPRICE + data << r.MEANPRICE + data << r.MAX_PRICE + data << r.TIME + data << r.FIRSTDATAT + data << r.THIRDDATA + data << r.RELATIVEERROR + data << r.SKEWNESS + data << r.PQ_PRICE_RANGE + data << r.DISTRICT_PRICE_RANGE + data << r.DESCRIPTION + data << r.PRIV_BASE_PRICE + data << r.IS_ARTTFICIAL + data << r.PRIV_FINAL_PRICE + data << r.CREATE_DATE + data << r.PRICE + data << r.PRICE_SOURCE + data << r.CASE_MD_PRICE + data << r.CASE_AVG_PRICE + data << r.INDEX_LABEL + data << r.MAIN_PROPERTY + data << r.COMMTYPE + data << r.NEW_DEAL_PRICE + data << r.NEW_DEAL_DATE + data << r.PRICE_MOM_RATIO + data << r.MAX_MOM_RATIO + data << r.PRICE_YOY_RATIO + data << r.SATUR_RATIO + data << r.NEIGHBOR_PRICE + allRows << data + }); + println allRows.size() + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); +// } + println "community_p_describe_gj 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "community_p_describe_gj 上传Ftp完成......" + } + + def exportCommCaseDetailGj(month){ + +// def citys = CcbCommonUtils.citys; + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def fileName = "/ccb/MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] + + for(def city : citys){ + def cityCode = cityCodeDataMap.get(city) + def sql = "select ID, CITYID, COMMUNITY_ID, NETLISTING_ID, PRICE_DATE, to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE from comm_case_detail_gj where cityid='${cityCode}' and price_date=${month}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.COMMUNITY_ID + data << r.NETLISTING_ID + data << r.PRICE_DATE + data << r.CREATE_DATE + allRows << data + if(allRows.size()%200000==0){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + allRows.clear() + } + }); + if(allRows.size()>0){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + } + } + + println "comm_case_detail_gj 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "comm_case_detail_gj 上传Ftp完成......" + } + + def exportLastFile(){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def dataCodeFileName = "/ccb/${curDate}.ok" + if(CommonDatautils.isWindow()){ + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok" + } + FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload(null, dataCodeFileName, curDate) + } + + def exportOkFile(){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def dataCodeFileName = "/ccb/${curDate}.ok" + if(CommonDatautils.isWindow()){ + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok" + } + FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload(null, dataCodeFileName, curDate) + } + + def exportInOk(){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + Vector files = CcbFtpTestUtils.listFileNames(curDate) + Iterator fileVec = files.iterator() + def fileRows = [] + while(fileVec.hasNext()){ + ChannelSftp.LsEntry file = fileVec.next() + + if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){ + fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(file.getFilename(), curDate) + } + } + def fileName = "/ccb/${curDate}.ok" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok" + } + FileUtils.createTxtFile(fileRows, fileName, "utf-8", false); + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "统计文件长度ok文件 上传Ftp完成......" + } +} + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy index f8770132c..19bf6561b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy @@ -6,6 +6,7 @@ import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.lixue.util.DBUtils import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject import java.util.concurrent.CopyOnWriteArrayList @@ -21,6 +22,66 @@ class AddressCut extends BaseBatch{ @Override public void process(Object args) { + sameAddr() + } + + def sameAddr() { + def citys = CommonDatautils.findAllCity() + def dataMap = [:] + def index = 0 + for (def c : citys) { + index++ + def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city", "region", "name", "address"]) + + def addrMap = [:] + for (def dist : dists) { + if (dist.address) { + def rets = addrMap.get(dist.address) + if (!rets) { + rets = [] + } + rets << dist + addrMap.put(dist.address, rets) + } + } + dataMap.put(c, addrMap) + println index +" "+c + } + + + def sameAddrList = [] + for(def c : citys) { + def cityMap = dataMap.get(c) + if(cityMap){ + cityMap.each{addr,rets-> + if(rets && rets.size()>1){ + def keys = new HashSet() + for(def d : rets){ + if(d.name){ + def name = d.name.replace("一期","").replace("二期","").replace("三期","").replace("四期","").replace("五期","").replace("六期","").replace("七期","").replace("八期","").replace("九期","").replace("十期","").replace("一区","").replace("二区","").replace("三区","").replace("四区","").replace("五区","").replace("六区","").replace("七区","").replace("八区","").replace("九区","").replace("十区","").replace("A区","").replace("B区","").replace("C区","").replace("D区","") + .replace("十一街区","").replace("十二街区","").replace("十三街区","").replace("十四街区","").replace("十五街区","").replace("十六街区","").replace("十七街区","").replace("十八街区","").replace("十九街区","").replace("一街区","").replace("二街区","").replace("三街区","").replace("四街区","").replace("五街区","").replace("六街区","").replace("七街区","").replace("八街区","").replace("九街区","").replace("十街区","") + .replace("公寓","").replace("别墅","").replace("商住楼","").replace("花园","").replace("南区","").replace("北区","").replace("东区","").replace("西区","") + keys << d.region +"##"+ name + } + } + if(keys && keys.size()>1){ + for(def d : rets) { + if (d.name) { + sameAddrList << [d.city, d.region, d.name, d.address] + } + } + } + } + } + } + } + //ExcelUtils.write("D:\\数据\\小区\\同地址小区.xlsx", sameAddrList, "相同地址小区") + FileUtils.batchCreateFile(sameAddrList, "D:\\数据\\小区\\同地址小区.txt", null, "utf-8", false); + println "完成" + } + + + def addrCut(){ def citys = CommonDatautils.findAllCity() def index = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/SSHHelperTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/SSHHelperTest.groovy new file mode 100644 index 000000000..0e94c9551 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/SSHHelperTest.groovy @@ -0,0 +1,112 @@ +package com.fangjia.script.ccbevaluation.ronghe + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; + +import com.jcraft.jsch.ChannelExec; +import com.jcraft.jsch.JSch; +import com.jcraft.jsch.Session + +import com.jcraft.jsch.*; + + +class SSHHelperTest { + + private Session session = null; + private Channel channel = null; + + private String sftpHost = "192.168.0.86"; + private int sftpPort = 17495; + private String sftpUserName = "ccb"; + private String sftpPassword = "ccb!@#"; + private int timeout = 30000; + /** + * 获取连接 + * @return + */ + private ChannelExec getChannelExec() { + try { + if (channel != null && channel.isConnected()) { + return (ChannelExec) channel; + } + JSch jSch = new JSch(); + if (session == null || !session.isConnected()) { + session = jSch.getSession(sftpUserName, sftpHost, sftpPort); + session.setPassword(sftpPassword); + Properties config = new Properties(); + config.put("StrictHostKeyChecking", "no"); + session.setConfig(config); + session.setTimeout(timeout); + session.connect(); + } + channel = session.openChannel("exec"); + } catch (Exception e) { + if (session != null) { + session.disconnect(); + session = null; + } + channel = null; + } + return channel == null ? null : (ChannelExec) channel; + } + + /** + * 关闭连接 + */ + private void closeChannel() { + try { + if (channel != null) { + channel.disconnect(); + channel = null; + } + if (session != null) { + session.disconnect(); + session = null; + } + } catch (Exception e) { + System.out.println(e); + } + } + + /** + * 执行服务器端命令 + */ + public boolean executeCommand(String command) { + boolean flag = false; + ChannelExec channelExec = getChannelExec(); + if (channelExec == null) { + return false; + } + try { + channelExec.setInputStream(null); + channelExec.setErrStream(System.err); + channelExec.setCommand(command); + + InputStream ins = channelExec.getInputStream(); + channelExec.connect(); + + BufferedReader reader = new BufferedReader(new InputStreamReader(ins, "utf-8")); + String line = null; + while ((line = reader.readLine()) != null) { + System.out.println(line); + } + + reader.close(); + closeChannel(); + + flag = true; + } catch (Exception e) { + System.out.println(e); + flag = false; + } + return flag; + } + + public static void main(String[] args) { + SSHHelperTest jschCommand = new SSHHelperTest(); + System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/package.sh 20220110")); + } + + +} From 7871c67c82ff8fb87f73bdd867b5c5e31a3dd385 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 28 Oct 2022 16:24:59 +0800 Subject: [PATCH 013/112] update --- .../ccbevaluation/ExportOracleDataTest.groovy | 64 +++++++++---------- .../script/gj/paimai/PaimaiDataSend.groovy | 10 +-- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy index a93ceb174..945ab6a17 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy @@ -32,41 +32,41 @@ class ExportOracleDataTest { def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") def new_dists = [] //不会变 10.25 - exportEvaluationRateData() - exportBigdataPoiData()// - exportBigdataDistrictData()// - - exportGuidedPrice()//随人工改变// - exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策// - - //处理小区情况 - exportBigdataCommunity()// - exportCommunityStaging()//没cityid - exportCommunityBuileType()// - - exportThirdPartyPrice2(lastMonth, new_dists)// - exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 - exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 - - //第三方网站权重表 - exportWebsiteRate(); - - //计算结果相关 - exportAvmEvaluationGJNB(curMonth)//无城市区分 - exportAVMEvaluationItemGJNB(curMonth)//无城市区分 - - exportCityPrice(curMonth)//无城市区分 - exportRegionPrice(curMonth)//无城市区分 - - exportCommunityBasePriceGj(curMonth)//已合下面的合并 //无城市区分 - exportCommunityBasePrice(curMonth)//无城市区分 - - exportCommunityPdescribeGj(curMonth)//无城市区分 - exportCommCaseDetailGj(curMonth)//无城市区分 +// exportEvaluationRateData() +// exportBigdataPoiData()// +// exportBigdataDistrictData()// +// +// exportGuidedPrice()//随人工改变// +// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策// +// +// //处理小区情况 +// exportBigdataCommunity()// +// exportCommunityStaging()//没cityid +// exportCommunityBuileType()// +// +// exportThirdPartyPrice2(lastMonth, new_dists)// +// exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 +// exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 +// +// //第三方网站权重表 +// exportWebsiteRate(); +// +// //计算结果相关 +// exportAvmEvaluationGJNB(curMonth)//无城市区分 +// exportAVMEvaluationItemGJNB(curMonth)//无城市区分 +// +// exportCityPrice(curMonth)//无城市区分 +// exportRegionPrice(curMonth)//无城市区分 +// +// exportCommunityBasePriceGj(curMonth)//已合下面的合并 //无城市区分 +// exportCommunityBasePrice(curMonth)//无城市区分 +// +// exportCommunityPdescribeGj(curMonth)//无城市区分 +// exportCommCaseDetailGj(curMonth)//无城市区分 def curDate = curMonth.toString()+"28" // ok文件 -// exportInOk() + exportInOk() SSHHelperTest jschCommand = new SSHHelperTest(); System.out.println(jschCommand.executeCommand("sh /home/ccb/output/eval_package.sh ${curDate}")); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy index 83ceb7171..cfc3ad6d2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataSend.groovy @@ -38,8 +38,8 @@ class PaimaiDataSend { // task.jdRunByMongo() // task.aliZcRun() - task.aliZcRun2() -// task.aliFpRun() +// task.aliZcRun2() + task.aliFpRun() // task.rmfysszcRun() // task.jdRun() // task.checkUrl() @@ -334,10 +334,10 @@ class PaimaiDataSend { def index = 0 def q = new BasicDBObject() // q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"])) - q.put("province","江苏") - q.put("sub_type","土地") +// q.put("province","江苏") +// q.put("sub_type","土地") // q.put("parser_status", new BasicDBObject('$ne',2)) - q.put("url","https://susong-item.taobao.com/auction/671320162557.htm") + q.put("url","https://sf-item.taobao.com/sf_item/673646300374.htm") //https://sf-item.taobao.com/sf_item/657131607486.htm // q.put("parser_status", new BasicDBObject('$exists',false)) // q.put("finishDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-09-01"))) From 57ca8daf07787db4666a50dc6351f4604f521372 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 28 Oct 2022 17:13:28 +0800 Subject: [PATCH 014/112] =?UTF-8?q?=E7=9B=AE=E5=BD=95=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/ExportOracleDataTest.groovy | 56 +++++++++---------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy index 945ab6a17..8a8abaf10 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy @@ -68,7 +68,7 @@ class ExportOracleDataTest { // ok文件 exportInOk() SSHHelperTest jschCommand = new SSHHelperTest(); - System.out.println(jschCommand.executeCommand("sh /home/ccb/output/eval_package.sh ${curDate}")); + System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}")); } @@ -91,7 +91,7 @@ class ExportOracleDataTest { }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; - def fileName = "/ccb/MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" } @@ -122,7 +122,7 @@ class ExportOracleDataTest { }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; - def fileName = "/ccb/MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat" } @@ -157,7 +157,7 @@ class ExportOracleDataTest { }); } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; - def fileName = "/ccb/MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" } @@ -185,7 +185,7 @@ class ExportOracleDataTest { allRows << data }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; - def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" } @@ -220,7 +220,7 @@ class ExportOracleDataTest { def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; - def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" } @@ -259,7 +259,7 @@ class ExportOracleDataTest { } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; - def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" } @@ -299,7 +299,7 @@ class ExportOracleDataTest { } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; - def fileName = "/ccb/MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" } @@ -336,7 +336,7 @@ class ExportOracleDataTest { } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; - def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" } @@ -381,7 +381,7 @@ class ExportOracleDataTest { } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; - def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" } @@ -502,7 +502,7 @@ class ExportOracleDataTest { allRows << data }) def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; - def fileName = "/ccb/MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" } @@ -519,7 +519,7 @@ class ExportOracleDataTest { def exportThirdPartyPrice2(price_date, new_dists) { def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; - def fileName = "/ccb/MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" // fileName = "D:\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" @@ -627,7 +627,7 @@ class ExportOracleDataTest { } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; - def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" } @@ -643,7 +643,7 @@ class ExportOracleDataTest { def exportBigdataForsale(publishDate, new_dists){ //["北京","上海"] def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; - def fileName = "/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" } @@ -809,8 +809,8 @@ class ExportOracleDataTest { }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; - def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" - def fileName1 = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = "/ccb_evaluation/MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" @@ -861,8 +861,8 @@ class ExportOracleDataTest { }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; - def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" - def fileName1 = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = "/ccb_evaluation/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat" @@ -904,8 +904,8 @@ class ExportOracleDataTest { }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; - def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" - def fileName1 = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" @@ -947,7 +947,7 @@ class ExportOracleDataTest { }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; - def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" } @@ -976,7 +976,7 @@ class ExportOracleDataTest { }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" - def fileName = "/ccb/MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat" } @@ -1007,7 +1007,7 @@ class ExportOracleDataTest { }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" - def fileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat" } @@ -1023,7 +1023,7 @@ class ExportOracleDataTest { // def citys = CcbCommonUtils.citys; // def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" - def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" } @@ -1093,7 +1093,7 @@ class ExportOracleDataTest { def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" - def fileName = "/ccb/MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" } @@ -1129,7 +1129,7 @@ class ExportOracleDataTest { def exportLastFile(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" - def dataCodeFileName = "/ccb/${curDate}.ok" + def dataCodeFileName = "/ccb_evaluation/${curDate}.ok" if(CommonDatautils.isWindow()){ dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok" } @@ -1139,7 +1139,7 @@ class ExportOracleDataTest { def exportOkFile(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" - def dataCodeFileName = "/ccb/${curDate}.ok" + def dataCodeFileName = "/ccb_evaluation/${curDate}.ok" if(CommonDatautils.isWindow()){ dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok" } @@ -1159,7 +1159,7 @@ class ExportOracleDataTest { fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(file.getFilename(), curDate) } } - def fileName = "/ccb/${curDate}.ok" + def fileName = "/ccb_evaluation/${curDate}.ok" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok" } From 2122bf384760d4b9860ef240ed34e483f50c09a0 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 28 Oct 2022 17:20:18 +0800 Subject: [PATCH 015/112] update --- .../ccbevaluation/CcbFtpTestUtils.groovy | 2 +- .../ccbevaluation/ExportOracleDataTest.groovy | 62 +++++++++---------- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy index b9f4572e5..2a9ad549f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy @@ -11,7 +11,7 @@ class CcbFtpTestUtils { static def ftpPort = 17495 static def ftpUser = "ccb" static def ftpPassword = "ccb!@#" - static def ftpPath = "/home/ccb/output/" + static def ftpPath = "/home/ccb_evaluation/output/" static def ftpPath_rh = "/home/ccb_rh/output/" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy index 8a8abaf10..431fc02c2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy @@ -32,37 +32,37 @@ class ExportOracleDataTest { def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") def new_dists = [] //不会变 10.25 -// exportEvaluationRateData() -// exportBigdataPoiData()// -// exportBigdataDistrictData()// -// -// exportGuidedPrice()//随人工改变// -// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策// -// -// //处理小区情况 -// exportBigdataCommunity()// -// exportCommunityStaging()//没cityid -// exportCommunityBuileType()// -// -// exportThirdPartyPrice2(lastMonth, new_dists)// -// exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 -// exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 -// -// //第三方网站权重表 -// exportWebsiteRate(); -// -// //计算结果相关 -// exportAvmEvaluationGJNB(curMonth)//无城市区分 -// exportAVMEvaluationItemGJNB(curMonth)//无城市区分 -// -// exportCityPrice(curMonth)//无城市区分 -// exportRegionPrice(curMonth)//无城市区分 -// -// exportCommunityBasePriceGj(curMonth)//已合下面的合并 //无城市区分 -// exportCommunityBasePrice(curMonth)//无城市区分 -// -// exportCommunityPdescribeGj(curMonth)//无城市区分 -// exportCommCaseDetailGj(curMonth)//无城市区分 + exportEvaluationRateData() + exportBigdataPoiData()// + exportBigdataDistrictData()// + + exportGuidedPrice()//随人工改变// + exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策// + + //处理小区情况 + exportBigdataCommunity()// + exportCommunityStaging()//没cityid + exportCommunityBuileType()// + + exportThirdPartyPrice2(lastMonth, new_dists)// + exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 + exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 + + //第三方网站权重表 + exportWebsiteRate(); + + //计算结果相关 + exportAvmEvaluationGJNB(curMonth)//无城市区分 + exportAVMEvaluationItemGJNB(curMonth)//无城市区分 + + exportCityPrice(curMonth)//无城市区分 + exportRegionPrice(curMonth)//无城市区分 + + exportCommunityBasePriceGj(curMonth)//已合下面的合并 //无城市区分 + exportCommunityBasePrice(curMonth)//无城市区分 + + exportCommunityPdescribeGj(curMonth)//无城市区分 + exportCommCaseDetailGj(curMonth)//无城市区分 def curDate = curMonth.toString()+"28" // ok文件 From 110118291e34355de6d8f3205be1e943f34c970a Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Fri, 28 Oct 2022 18:45:01 +0800 Subject: [PATCH 016/112] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=AD=A6=E6=A0=A1?= =?UTF-8?q?=E7=BB=8F=E7=BA=AC=E5=BA=A6=E8=8E=B7=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/poi/FillLngLat.groovy | 20 ++++---- .../temp/shenjian/util/BaiDuUtils.groovy | 48 +++++++++++++++++++ 2 files changed, 59 insertions(+), 9 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/BaiDuUtils.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/FillLngLat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/FillLngLat.groovy index a0c3ec24f..3c5c2a52d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/FillLngLat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/FillLngLat.groovy @@ -1,6 +1,7 @@ package com.fangjia.script.temp.shenjian.poi import com.alibaba.fastjson.JSONObject +import com.fangjia.script.temp.shenjian.util.BaiDuUtils import com.fangjia.script.temp.shenjian.util.EsUtils import com.fangjia.script.temp.shenjian.util.GaoDeUtils import com.fangjia.script.util.ElasticUtils @@ -15,25 +16,26 @@ import java.util.concurrent.Executors */ class FillLngLat { - static ExecutorService executorService = Executors.newFixedThreadPool(4) +// static ExecutorService executorService = Executors.newFixedThreadPool(4) static void main(String[] args) { - ElasticUtils.executeSqlQuery("select * from dws_poi where location is not null", { dataT -> + ElasticUtils.executeSqlQuery("select * from dws_poi where location is null", { dataT -> final def data = dataT - executorService.submit(new Runnable() { - @Override - void run() { +// executorService.submit(new Runnable() { +// @Override +// void run() { println(data) - def lngLat = GaoDeUtils.parserByGaode(data.city, data.region + data.name + data.address) + def lngLat = BaiDuUtils.parserByBaidu(data.city, data.region + data.name + data.address) + Thread.sleep(1000) if (lngLat != null && lngLat.size() == 2 ) { JSONObject jsonObject = new JSONObject() jsonObject.put("lon", Double.valueOf(lngLat['lng'])) jsonObject.put("lat", Double.valueOf(lngLat['lat'])) EsUtils.bulkUpdateFieldByIds("dws_poi", "location", jsonObject, Arrays.asList(data.source_id)) } - } - }) +// } +// }) }) - executorService.shutdown() + //executorService.shutdown() } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/BaiDuUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/BaiDuUtils.groovy new file mode 100644 index 000000000..e307bf28a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/BaiDuUtils.groovy @@ -0,0 +1,48 @@ +package com.fangjia.script.temp.shenjian.util + +import com.alibaba.fastjson.JSONObject +import com.fangjia.core.util.GeoUtils +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.util.DataUtil +import org.apache.commons.lang.StringUtils + +/** + * 百度工具类 + * + * @author shenjian + * @since 2022/10/28 + */ +class BaiDuUtils { + + def static final String KEY = "6w9SEHS3hnAFjmY2oEhqIDdgvLPKWXu5" + + /** + * 百度地址逆解析,返回高德坐标!!!!!!!!!!!!!!! + * @param city + * @param address + */ + def static parserByBaidu(def city, def address) { + Map params = new HashMap() + params.put("city", city) + if (StringUtils.isBlank(address)) { + return [] + } + params.put("address", address.replaceAll("null", "")) + params.put("output", "json") + params.put("ak", KEY) + params.put("callback", "showLocation") + + String content + content = new HttpRequest().get("https://api.map.baidu.com/geocoding/v3/", params, "utf-8") + //content = "showLocation&&showLocation({\"status\":0,\"result\":{\"location\":{\"lng\":131.04009309945693,\"lat\":45.78426770180344},\"precise\":1,\"confidence\":70,\"comprehension\":100,\"level\":\"地产小区\"}})" + content = content.replaceAll("showLocation&&showLocation\\(", "").replaceAll("\\)", "") + JSONObject json = JSONObject.parseObject(content) + if (json.status == 0 && json.result && json.result.location) { + def bd_lng = json.result.location.lng + def bd_lat = json.result.location.lat + Map locationMap = GeoUtils.bd09ToGcj02(bd_lat, bd_lng) + return ["lng": com.fangjia.core.util.DataUtil.getDoubleRound(6, locationMap['lng']), "lat": com.fangjia.core.util.DataUtil.getDoubleRound(6, locationMap['lat'])] + } + return [] + } +} From b5750571d18099caeeef13c3afc6e963b9f4f08e Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sat, 29 Oct 2022 13:09:49 +0800 Subject: [PATCH 017/112] =?UTF-8?q?=E6=B5=8B=E8=AF=9586=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ExportOracleAllDataTest.groovy | 1194 +++++++++++++++++ 1 file changed, 1194 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy new file mode 100644 index 000000000..145aad4eb --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy @@ -0,0 +1,1194 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.core.RegionData +import com.fangjia.script.ccbevaluation.ronghe.SSHHelper +import com.fangjia.script.ccbevaluation.ronghe.SSHHelperTest +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.jcraft.jsch.ChannelSftp +import groovy.sql.Sql +import org.apache.commons.codec.digest.DigestUtils + +import java.sql.ResultSet + +class ExportOracleAllDataTest { + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + def citys = null + + static main(args) { + def task = new ExportOracleAllDataTest() + task.run(args) + +// citys = CcbCommonUtils.citys_252 +// def rows = [] +// for(def c : citys){ +// def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city","region", "name"]) +// rows << [c, dists.size()] +// } +// ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\txt\\252城小区量统计.xlsx", rows, "数量") + } + + def run(args) { + + citys = CcbCommonUtils.getRunningCitys(args)//["潍坊","济宁","威海","临沂","绵阳","陇南","塔城","阿勒泰","迪庆州","日喀则"] //CcbCommonUtils.getRunningCitys(args) + def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); + def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") + def new_dists = [] + //不会变 10.25 + exportEvaluationRateData() + exportBigdataPoiData()// + exportBigdataDistrictData()// + + exportGuidedPrice()//随人工改变// + exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策// + + //处理小区情况 + exportBigdataCommunity()// + exportCommunityStaging()//没cityid + exportCommunityBuileType()// + + exportThirdPartyPrice2(new_dists)// + exportBigdataForsale(new_dists) //2022-07-19 + exportBigdataDeal( new_dists) //2022-07-01 + + //第三方网站权重表 + exportWebsiteRate(); + + //计算结果相关 + exportAvmEvaluationGJNB()// + exportAVMEvaluationItemGJNB()// + + exportCityPrice()// + exportRegionPrice()// + + exportCommunityBasePriceGj()// + exportCommunityBasePrice()// + + exportCommunityPdescribeGj()// + exportCommCaseDetailGj()// + + def curDate = curMonth.toString()+"28" + // ok文件 + exportInOk() + SSHHelperTest jschCommand = new SSHHelperTest(); + System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}")); + + } + + + def exportEvaluationRateData() { + def sql = "select ID,RATE_TYPE,RATE,RANK,CITY,REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from evaluate_rate_gj"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.RATE_TYPE + data << r.RATE + data << r.RANK + data << r.CITY + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["系数ID","系数类型","系数","排序","城市","备注","创建时间","更新时间","来源公司"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "上传Ftp完成......" + } + + + def exportWebsiteRate(){ + def sql = "select DATA_SOURCE_ID,DATA_SOURCE,DATA_SOURCE_MAP,WEIGHT,TYPE,SOURCECODE,CITYID,CITYNAME,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from WEBSITE_CONFIG_GJ"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.DATA_SOURCE_ID + data << r.DATA_SOURCE + data << r.DATA_SOURCE_MAP + data << r.WEIGHT + data << r.TYPE + data << r.SOURCECODE + data << r.CITYID + data << r.CITYNAME + data << r.CREATETIME + data << r.UPDATETIME + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["系数ID","系数类型","系数","排序","城市","备注","创建时间","更新时间","来源公司"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "上传Ftp完成......" + } + + + def exportBigdataPoiData() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID, DISTID, ID, POI_NAME, TYPE, TYPE_CODE, LATITUDE, LONGITUDE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from bigdata_poi_gj where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.CITYID + data << r.DISTID + data << r.ID + data << r.POI_NAME + data << r.TYPE + data << r.TYPE_CODE + data << r.LATITUDE + data << r.LONGITUDE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","来源公司"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "BIGDATA_POI_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "BIGDATA_POI_GJ 上传Ftp完成......" + } + + def exportCommunityStaging() { + + def allRows = [] + def sql = "select ID,COMMID,MASTER_COMMID,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_staging_gj"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMID + data << r.MASTER_COMMID + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["分期ID","小区ID","小区主期ID","备注","创建时间","更新时间","来源公司"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "COMMUNITY_STAGING_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "COMMUNITY_STAGING_GJ 上传Ftp完成......" + } + + def exportCommunityBuileType() { +// def citys = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"] +// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID, COMMID, BUILDTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, WORKDATE from bigdata_commbuildtype_gj where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.BUILDTYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.WORKDATE + allRows << data + }); + } + + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "BIGDATA_COMMBUILDTYPE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "BIGDATA_COMMBUILDTYPE_GJ 上传Ftp完成......" + } + + def exportBigdataDistrictData() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select DISTID, DISTNAME, DISTSPELL, CITYID, DISTCODE, SEQ, DISTTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE, LJXZQH, LJXZQHDM, XZQHDM, WORKDATE from bigdata_district_gj where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.DISTID + data << r.DISTNAME + data << r.DISTSPELL + data << r.CITYID + data << r.DISTCODE + data << r.SEQ + data << r.DISTTYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.LJXZQH + data << r.LJXZQHDM + data << r.XZQHDM + data << r.WORKDATE + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" + } + def headers = [] +//["行政区ID","行政区名称","行政区名称拼音","城市ID","国标码","显示顺序","行政区类别","备注","创建时间","更新时间","来源公司","逻辑行政区","逻辑行政区划代码","行政区划代码","数据时间"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "BIGDATA_DISTRICT_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "BIGDATA_DISTRICT_GJ 上传Ftp完成......" + } + + + def exportGuidedPrice() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE,AVG_PRICE, GUIDING_PERSON, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE from guided_price_gj where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMID + data << r.COMMNAME + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.DEALDATE + data << r.AVG_PRICE + data << r.GUIDING_PERSON + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" + } + def headers = [] +//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "GUIDED_PRICE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "GUIDED_PRICE_GJ 上传Ftp完成......" + } + + def exportCommunityGuidedPrice(def date) { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where CITYID='${cityCode}' and createtime>=to_date('${date}','yyyyMMdd')"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMID + data << r.COMMNAME + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.AVGPRICE + data << r.SOURCETYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导价格","类型","备注","创建时间","更新时间","来源公司"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "COMMUNITY_GUIDE_PRICE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "COMMUNITY_GUIDE_PRICE_GJ 上传Ftp完成......" + } + + def exportBigdataDeal(def new_dists) { + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + + "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + + "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + + "from bigdata_deal_gj where CITYID='${cityCode}' and distid not in ${newstr}" + def rets1 = dealData(sql1) + + def sql2 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + + "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + + "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + + "from bigdata_deal_gj where CITYID='${cityCode}' and distid in ${newstr}" + def rets2 = dealData(sql2) + allRows.addAll(rets1) + allRows.addAll(rets2) + }else{ + def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + + "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + + "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + + "from bigdata_deal_gj where CITYID='${cityCode}' "; + allRows.addAll(dealData(sql)) + } + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "BIGDATA_DEAL_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "BIGDATA_DEAL_GJ 上传Ftp完成......" + } + + + def dealData(def sql){ + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.CITYID + data << r.DEALID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.STREET + data << r.STREETNO + data << r.COMMID + data << r.COMMNAME + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.UNITID + data << r.UNITNO + data << r.UNITNAME + data << r.HOUSEID + data << r.ADDRESS + data << r.BUILDYEAR + data << r.PROPTYPE + data << r.PROPYEARS + data << r.PERIODNUM + data << r.FLOORNUM + data << r.FLOORTYPE + data << r.FLOORCNT + data << r.BUILDTYPE + data << r.BUILDSTRU + data << r.BUILDAREA + data << r.LIVINGAREA + data << r.HOUSESTYLE + data << r.ROOMCNT + data << r.LIVINGROOMCNT + data << r.KITCHENCNT + data << r.BATHROOMCNT + data << r.HOUSEUSE + data << r.HOUSESTRU + data << r.HOUSEASPECT + data << r.FITMENT + data << r.EVALUATENO + data << r.DEALUNITPRICE + data << r.DEALTOTALPRICE + data << r.DECLAREPRICE + data << r.EVALUATEPRICE + data << r.CONTRACTNO + data << r.DEALDATE + data << r.DEALTYPE + data << r.PAYTYPE + data << r.DEALPERIOD + data << r.IFELEVATOR + data << r.COMPAREAGARDEN + data << r.COMPAREATERRACE + data << r.COMPAREABALCONY + data << r.COMPAREABAYWINDOW + data << r.COMPAREAATTIC + data << r.COMPAREAGARAGE + data << r.COMPAREABASEMENT + data << r.COMPAREAOTHER + data << r.DATASOURCE + data << r.AGENCYID + data << r.AGENCYNAME + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.CJJE + data << r.WORKDATE + allRows << data + }); + return allRows + } + + def exportThirdPartyPrice() { + def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, ZHUGE_AVG_PRICE, ANJUKE_AVG_PRICE, FANG_AVG_PRICE, LIANJIA_AVG_PRICE, CITYHOUSE_AVG_PRICE, MAITIAN_AVG_PRICE, KE_AVG_PRICE, \n" + + "TAOFANG_AVG_PRICE, ZHONGYUAN_AVG_PRICE, TBS_AVG_PRICE, QFANG_AVG_PRICE, FANGTAN_AVG_PRICE, FANGDUODUO_AVG_PRICE, WIWJ_AVG_PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE\n" + + "from third_party_price_gj"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMID + data << r.COMMNAME + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.YEAR_MONTH + data << r.ZHUGE_AVG_PRICE + data << r.ANJUKE_AVG_PRICE + data << r.FANG_AVG_PRICE + data << r.LIANJIA_AVG_PRICE + data << r.CITYHOUSE_AVG_PRICE + data << r.MAITIAN_AVG_PRICE + data << r.KE_AVG_PRICE + data << r.TAOFANG_AVG_PRICE + data << r.ZHONGYUAN_AVG_PRICE + data << r.TBS_AVG_PRICE + data << r.QFANG_AVG_PRICE + data << r.FANGTAN_AVG_PRICE + data << r.FANGDUODUO_AVG_PRICE + data << r.WIWJ_AVG_PRICE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE.replaceAll(~'\\r', '') + allRows << data + }) + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] +//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "THIRD_PARTY_PRICE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "THIRD_PARTY_PRICE_GJ 上传数据文件完成......" + } + + + def exportThirdPartyPrice2(new_dists) { + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" +// fileName = "D:\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] + def index = 0 + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def allRows = [] + if(new_dists) { + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " + + "COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " + + "CITYID='${cityCode}' and distid not in ${newstr} "; + def rets1 = getThirdPartyDatas(sql1) + allRows.addAll(rets1) + + def sql2 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " + + "COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " + + "CITYID='${cityCode}' and distid in ${newstr} "; + def rets2 = getThirdPartyDatas(sql2) + allRows.addAll(rets2) + }else{ + def sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where CITYID='${cityCode}'"; + allRows = getThirdPartyDatas(sql) + } + if(allRows && allRows.size()>0){ + index = index + 1 + if(index==1){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + } + } + } + + println "BIGDATA_THIRD_PARTY_PRICE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + } + + def getThirdPartyDatas(def sql){ + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.CITYID + data << r.DISTID + data << r.COMMID + data << r.COMMNAME + data << r.DATA_SOURCE_ID + data << r.DATA_SOURCE + data << r.PRICE_DATE + data << r.COMMUNITY_PRICE + data << r.COMMUNITY_MAX_PRICE + data << r.COMMUNITY_MIN_PRICE + data << r.UPDATETIME + allRows << data + }) + return allRows; + } + + def exportBigdataCommunity(){ + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID, DISTID, COMMID, DISTNAME, COMMNAME, ADDRESS, LATITUDE, LONGITUDE, LANDAREA, BUILDAREA, GREENINGRATE, PLOTRATIO, PARKINGCNT, BUILDYEAR, HOUSEUSE, ZTS, WORKDATE, UPDATEFLAG, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE, \n" + + "\tCOMMALIAS, STREET, STREETNO, IFNEW, INTRODUCTION, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, SOURCECODE, XZJDB, LJXZQHDM, LJXZQH,STATUS from bigdata_community_gj where CITYID='${cityCode}' ";//where cityId in (350200,150100) + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.DISTID + data << r.COMMID + data << r.DISTNAME + data << r.COMMNAME + data << r.ADDRESS + data << r.LATITUDE + data << r.LONGITUDE + data << r.LANDAREA + data << r.BUILDAREA + data << r.GREENINGRATE + data << r.PLOTRATIO + data << r.PARKINGCNT + data << r.BUILDYEAR + data << r.HOUSEUSE + data << r.ZTS + data << r.WORKDATE + data << r.UPDATEFLAG + data << r.UPDATETIME + data << r.COMMTYPE + data << r.COMMALIAS + data << r.STREET + data << r.STREETNO + data << r.IFNEW + data << "" + data << r.REMARK + data << r.CREATETIME + data << r.SOURCECODE + data << r.XZJDB + data << r.LJXZQHDM + data << r.LJXZQH + data << r.STATUS + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途", + //"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司", + //"乡镇/街道办","逻辑行政区划代码","逻辑行政区划","状态"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "BIGDATA_COMMUNITY_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "BIGDATA_COMMUNITY_GJ 上传Ftp完成......" + } + + def exportBigdataForsale(new_dists){ + //["北京","上海"] + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"] + def index = 0 + for(def city : citys){ + def cityCode = cityCodeMap.get(city) + + def allRows = [] + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj " + + "where cityId='${cityCode}' and distid not in ${newstr}"; + def rets1 = getHouseDatas(sql1) + allRows.addAll(rets1) + }else{ + def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}' "; + allRows = getHouseDatas(sql) + } + if(allRows.size()>0){ + index = index + 1 + if(index == 1){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + } + + } + } + + if(new_dists){//补充被修区域的数据 + def newstr = "('"+new_dists.join("','")+"')" + def sql2 = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH " + + "from bigdata_forsale_gj where distid in ${newstr}" + def rets2 = getHouseDatas(sql2) + if(rets2.size()>0){ + FileUtils.batchCreateFile(rets2, fileName, headers, "utf-8", true); + } + } + println "BIGDATA_FORSALE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "BIGDATA_FORSALE_GJ 上传Ftp完成......" + } + + def getHouseDatas(sql){ + def allRows = [] + def index = 0 + sqlInstance.eachRow(sql,{r-> + index = index+1 + println r.CITYNAME +" "+index + def data = [] + data << r.CITYID + data << r.FORSALEID + data << r.PROPCODE + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.UNITID + data << r.UNITNO + data << r.UNITNAME + data << r.HOUSEID + data << r.ADDRESS + data << r.BUILDYEAR + data << r.PROPTYPE + data << r.PROPYEARS + data << r.UNITPRICE + data << r.TOTALPRICE + data << r.PAYTYPE + data << r.FLOORNUM + data << r.FLOORCNT + data << r.BUILDTYPE + data << r.HOUSESTRU + data << r.BUILDSTRU + data << r.BUILDAREA + data << r.LIVINGAREA + data << r.HOUSESTYLE + data << r.ROOMCNT + data << r.LIVINGROOMCNT + data << r.KITCHENCNT + data << r.BATHROOMCNT + data << r.HOUSEASPECT + data << r.FITMENT + data << r.IFELEVATOR + data << r.ELEVATORHOUSE + data << r.PUBLISHTIME + data << r.EXPIRETIME + data << r.IFFIRST + data << r.DATASOURCE + data << r.AGENCYID + data << r.AGENCYNAME + data << r.PUBLISHER + data << r.PHONE + data << "" + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.ATTACH + allRows << data +// if(allRows.size()%200000==0){ +// FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); +// allRows.clear() +// } + }); + return allRows + } + + + def exportAvmEvaluationGJNB(){ + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = "/ccb_evaluation/MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" + fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" + } + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID,CITYID,CITYNAME, YEAR_MONTH, COMMUNITY_NUM, HAS_PRICE_COMMUNITY_NUM, PRICE_COVERAGE, ARTIFICIAL_CALC_NUM, ARTIFICIAL_CALC_COVERAGE, SAMPLE_CALC_NUM, SAMPLE_CALC_COVERAGE, \n" + + "ANN_CALC_NUM, ANN_CALC_COVERAGE, TEST_NUM, TEST_COMM_NUM, TEST_COMM_COVERAGE, PPE15_COVERAGE, RATE_MEDIAN, COEFFICIENT,DEAL_TEST_CASE_NUM, DEAL_TEST_COMM_NUM, DEAL_TEST_COMM_COVERAGE, DEAL_PPE15_COVERAGE, \n" + + "DEAL_COEFFICIENT, DEAL_RATE_MEDIAN, THIRD_TEST_CASE_NUM, THIRD_TEST_COMM_NUM, THIRD_TEST_COMM_COVERAGE, THIRD_PPE15_COVERAGE, THIRD_COEFFICIENT, THIRD_RATE_MEDIAN, \n" + + "to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from avm_evaluation_gj where CITYID='${cityCode}' "; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.YEAR_MONTH + data << r.COMMUNITY_NUM + data << r.HAS_PRICE_COMMUNITY_NUM + data << r.PRICE_COVERAGE + data << r.ARTIFICIAL_CALC_NUM + data << r.ARTIFICIAL_CALC_COVERAGE + data << r.SAMPLE_CALC_NUM + data << r.SAMPLE_CALC_COVERAGE + data << r.ANN_CALC_NUM + data << r.ANN_CALC_COVERAGE + data << r.TEST_NUM + data << r.TEST_COMM_NUM + data << r.TEST_COMM_COVERAGE + data << r.PPE15_COVERAGE + data << r.RATE_MEDIAN + data << r.COEFFICIENT + data << r.DEAL_TEST_CASE_NUM + data << r.DEAL_TEST_COMM_NUM + data << r.DEAL_TEST_COMM_COVERAGE + data << r.DEAL_PPE15_COVERAGE + data << r.DEAL_COEFFICIENT + data << r.DEAL_RATE_MEDIAN + data << r.THIRD_TEST_CASE_NUM + data << r.THIRD_TEST_COMM_NUM + data << r.THIRD_TEST_COMM_COVERAGE + data << r.THIRD_PPE15_COVERAGE + data << r.THIRD_COEFFICIENT + data << r.THIRD_RATE_MEDIAN + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + + def headers = [] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false); + println "AVM_EVALUATION_GJNB 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + CcbFtpTestUtils.ftpUpload(null, fileName1, curDate) + println "AVM_EVALUATION_GJNB 上传Ftp完成......" + } + + def exportAVMEvaluationItemGJNB(){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = "/ccb_evaluation/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" + fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat" + } + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, CONFIDENCE,TARGET_AVG_PRICE, \n" + + "to_char(DEALDATE,'yyyy-mm-dd hh24:mi:ss') as DEALDATE, UNIT_PRICE, BUILDAREA, FITMENT, HOUSEASPECT, FLOORNUM, FLOORCNT, DATA_TYPE, RATE, PE, \n" + + "to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME from avm_evaluation_item_gj where CITYID='${cityCode}' "; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.AVG_PRICE + data << r.CALC_TYPE + data << r.CONFIDENCE + data << r.TARGET_AVG_PRICE + data << r.DEALDATE + data << r.UNIT_PRICE + data << r.BUILDAREA + data << r.FITMENT + data << r.HOUSEASPECT + data << r.FLOORNUM + data << r.FLOORCNT + data << r.DATA_TYPE + data << r.RATE + data << r.PE + data << r.CREATETIME + allRows << data + }); + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","CONFIDENCE","TARGET_AVG_PRICE","DEALDATE","UNIT_PRICE","BUILDAREA","FITMENT","HOUSEASPECT","FLOORNUM","FLOORCNT","DATA_TYPE","RATE","PE","CREATETIME"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false); + println "AVM_EVALUATION_ITEM_GJNB 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + CcbFtpTestUtils.ftpUpload(null, fileName1, curDate) + println "AVM_EVALUATION_ITEM_GJNB 上传Ftp完成......" + } + + def exportCommunityBasePrice(){ + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where CITYID='${cityCode}' "; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.AVG_PRICE + data << r.CALC_TYPE + data << r.FAKE_RATE + data << r.DIFF_RATE + data << r.CONFI_RATE + data << r.CORR_RATE + data << r.CONFIDENCE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false); + println "COMMUNITY_BASE_PRICE_GJNB 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + CcbFtpTestUtils.ftpUpload(null, fileName1, curDate) + println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......" + } + + def exportCommunityBasePriceGj(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where CITYID='${cityCode}' "; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.AVG_PRICE + data << r.CALC_TYPE + data << r.FAKE_RATE + data << r.DIFF_RATE + data << r.CONFI_RATE + data << r.CORR_RATE + data << r.CONFIDENCE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "COMMUNITY_BASE_PRICE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "COMMUNITY_BASE_PRICE_GJ 上传Ftp完成......" + } + + def exportCityPrice(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + + def sql = "select ID, CITYID, CITYNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from city_price_gj where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.PRICE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def fileName = "/ccb_evaluation/MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "CITY_PRICE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "CITY_PRICE_GJ 上传Ftp完成......" + } + + def exportRegionPrice(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from DISTRICT_PRICE_GJ where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.PRICE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def fileName = "/ccb_evaluation/MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "DISTRICT_PRICE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "DISTRICT_PRICE_GJ 上传Ftp完成......" + } + + def exportCommunityPdescribeGj(){ + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def fileName = "/ccb_evaluation/MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMUNITY_ID + data << r.CITYID + data << r.CITY_NAME + data << r.DISTRICT_CODE + data << r.DISTRICT_NAME + data << r.COMMUNITY_NAME + data << r.PQ_NAME + data << r.BASE_PRICE + data << r.CONFIDENCE_LEVEL + data << r.CNT + data << r.MIN_PRICE + data << r.MEDIANPRICE + data << r.MEANPRICE + data << r.MAX_PRICE + data << r.TIME + data << r.FIRSTDATAT + data << r.THIRDDATA + data << r.RELATIVEERROR + data << r.SKEWNESS + data << r.PQ_PRICE_RANGE + data << r.DISTRICT_PRICE_RANGE + data << r.DESCRIPTION + data << r.PRIV_BASE_PRICE + data << r.IS_ARTTFICIAL + data << r.PRIV_FINAL_PRICE + data << r.CREATE_DATE + data << r.PRICE + data << r.PRICE_SOURCE + data << r.CASE_MD_PRICE + data << r.CASE_AVG_PRICE + data << r.INDEX_LABEL + data << r.MAIN_PROPERTY + data << r.COMMTYPE + data << r.NEW_DEAL_PRICE + data << r.NEW_DEAL_DATE + data << r.PRICE_MOM_RATIO + data << r.MAX_MOM_RATIO + data << r.PRICE_YOY_RATIO + data << r.SATUR_RATIO + data << r.NEIGHBOR_PRICE + allRows << data + }); + } + + println allRows.size() + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "community_p_describe_gj 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "community_p_describe_gj 上传Ftp完成......" + } + + def exportCommCaseDetailGj(){ + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def fileName = "/ccb_evaluation/MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] + + def index = 0 + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select ID, CITYID, COMMUNITY_ID, NETLISTING_ID, PRICE_DATE, to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE from comm_case_detail_gj where cityid='${cityCode}'"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.CITYID + data << r.COMMUNITY_ID + data << r.NETLISTING_ID + data << r.PRICE_DATE + data << r.CREATE_DATE + allRows << data + }); + if (allRows && allRows.size() > 0) { + index = index + 1 + if (index == 1) { + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + } else { + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + } + } + } + println "comm_case_detail_gj 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "comm_case_detail_gj 上传Ftp完成......" + } + + def exportLastFile(){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def dataCodeFileName = "/ccb_evaluation/${curDate}.ok" + if(CommonDatautils.isWindow()){ + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok" + } + FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload(null, dataCodeFileName, curDate) + } + + def exportOkFile(){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def dataCodeFileName = "/ccb_evaluation/${curDate}.ok" + if(CommonDatautils.isWindow()){ + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok" + } + FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload(null, dataCodeFileName, curDate) + } + + def exportInOk(){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + Vector files = CcbFtpTestUtils.listFileNames(curDate) + Iterator fileVec = files.iterator() + def fileRows = [] + while(fileVec.hasNext()){ + ChannelSftp.LsEntry file = fileVec.next() + + if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){ + fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(file.getFilename(), curDate) + } + } + def fileName = "/ccb_evaluation/${curDate}.ok" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok" + } + FileUtils.createTxtFile(fileRows, fileName, "utf-8", false); + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "统计文件长度ok文件 上传Ftp完成......" + } +} + From ea0079b28402828030a4d088dc0b7ce0f8798994 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 31 Oct 2022 16:51:54 +0800 Subject: [PATCH 018/112] =?UTF-8?q?=E5=B0=8F=E5=8C=BA=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?=E9=94=99=E8=AF=AF=E5=AD=97=E6=AE=B5=E8=BE=93=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/com/fangjia/dal/AbstractMongoDao.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/AbstractMongoDao.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/AbstractMongoDao.java index f985f20d0..379e46498 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/AbstractMongoDao.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/AbstractMongoDao.java @@ -415,7 +415,7 @@ public abstract class AbstractMongoDao extends AbstractJp try { writeMethod.invoke(entity, value); } catch (IllegalArgumentException e) { - logger.error("Error set value to " + value + "[" + value.getClass() + "] of field " + logger.error("["+key+"] Error set value to " + value + "[" + value.getClass() + "] of field " + descriptor.getName() + "[" + descriptor.getPropertyType() + "], class:" + entity.getClass().getSimpleName()); throw new RuntimeException("Error set value to " + value + "[" + value.getClass() + "] of field " + descriptor.getName() + "[" + descriptor.getPropertyType() + "]", e); From fbaadf919bed8e33ce939545f8dc12fb9a76856d Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 31 Oct 2022 21:25:22 +0800 Subject: [PATCH 019/112] =?UTF-8?q?=E6=B8=A4=E6=B5=B7=E6=96=B0=E5=9C=B0?= =?UTF-8?q?=E5=9D=80=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../weixin/WeiXinEvaluationController.java | 5 +- .../com/fangjia/open/util/BHBankUtils.java | 72 +++++++++---------- 2 files changed, 36 insertions(+), 41 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java index 078ca3b0f..de0c0040b 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java @@ -481,7 +481,7 @@ public class WeiXinEvaluationController { if(o != null){ String bhDistName = (String)o.get("name"); - String regionCode = String.valueOf((Long)o.get("country")); + String regionCode = String.valueOf(o.get("country")); String bhRegion = BHBankUtils.regionCodeMap.get(regionCode); bhMatchMap.put("preCode", (String)o.get("code")); bhMatchMap.put("preName", bhDistName); @@ -623,8 +623,6 @@ public class WeiXinEvaluationController { model.setEvaluationAudit(evaluationAudit); if(fjtUser!=null && fjtUser.getRole()==17 && district.getCity().equals("上海")){ - - Map retMap = getLoanInfo(district, evaluation.getHouseArea()); if(retMap!=null && retMap.size()>0){ model.setLoanInfoMap(retMap); @@ -678,7 +676,6 @@ public class WeiXinEvaluationController { } return model; } - //To C 报告结果页 public Object resultAction(EvaluationModel model){ String evaluationId = model.getEvaluationId(); diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java b/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java index 2c115f3b4..3ae49cc24 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java @@ -17,54 +17,51 @@ import java.util.*; public class BHBankUtils { -// public static String token = "bohaifangjiatest";//测试token -// public static String uri = "http://topen.shbrea.com.cn/API";//测试 +// public static String token = "bh";// 原:bohaifangjiatest +// public static String uri = "http://shbreac.ruitingcc.com/API/";//测试 原:http://topen.shbrea.com.cn/API public static String token = "bohaifangjia";//正式token - public static String uri ="https://open.shbrea.com.cn/API"; + public static String uri = "http://bizapi.shbrea.com.cn/API"; //"https://open.shbrea.com.cn/API"; //正式 http://bizapi.shbrea.com.cn/API/ public static String nonce = "818975"; public static Map regionCodeMap = new HashMap(){{ - put("310101000000","黄浦"); - put("310104000000","徐汇"); - put("310105000000","长宁"); - put("310106000000","静安"); - put("310107000000","普陀"); - put("310109000000","虹口"); - put("310110000000","杨浦"); - put("310112000000","闵行"); - put("310113000000","宝山"); - put("310114000000","嘉定"); - put("310115000000","浦东"); - put("310116000000","金山"); - put("310117000000","松江"); - put("310118000000","青浦"); - put("310120000000","奉贤"); - put("310230000000","崇明"); + put("11","黄浦"); + put("14","徐汇"); + put("12","长宁"); + put("13","静安"); + put("15","普陀"); + put("16","虹口"); + put("17","杨浦"); + put("19","闵行"); + put("20","宝山"); + put("21","嘉定"); + put("18","浦东"); + put("26","金山"); + put("25","松江"); + put("24","青浦"); + put("23","奉贤"); + put("22","崇明"); }}; public static List getDistrictByBhbank(String address){ List list = new ArrayList(); try { + Map params = new HashMap(); + params.put("keyword",address); -// Thread.sleep(20*1000); + Integer date1 = (int)(new Date().getTime()/1000); + String str = token + date1.toString()+nonce; + String md5str2 = MD5Util.encrypt(str,"utf-8"); - Map params = new HashMap(); - params.put("keyword",address); + Map headers = new HashMap(); + headers.put("timestamp", date1.toString()); + headers.put("sign", md5str2); + headers.put("nonce", nonce.toString()); - Integer date1 = (int)(new Date().getTime()/1000); - String str = token + date1.toString()+nonce; - String md5str2 = MD5Util.encrypt(str,"utf-8"); - - Map headers = new HashMap(); - headers.put("timestamp", date1.toString()); - headers.put("sign", md5str2); - headers.put("nonce", nonce.toString()); - HttpRequest request = new HttpRequest(); - String content = null; - content = request.get(uri +"/searchAddr/bh/", params, headers, "utf-8"); + HttpRequest request = new HttpRequest(); + String content = request.get(uri +"/searchAddr/bh/", params, headers, "utf-8"); System.out.println("/searchAddr/bh/ :"+content); if(StringUtils.isNotBlank(content)){ JSONObject jsonObj = JSONObject.fromObject(content); @@ -477,11 +474,12 @@ public class BHBankUtils { System.out.println("完成调用渤海转评估报告 "+content); if(StringUtils.isNotBlank(content)){ JSONObject jsonObj = JSONObject.fromObject(content); - Integer code = (Integer) jsonObj.get("errCode"); - if (code == 0) { - return code; + if(jsonObj.containsKey("errCode")){ + Integer code = (Integer) jsonObj.get("errCode"); + if (code == 0) { + return code; + } } - } }catch (Exception e){ e.printStackTrace(); From 095266b39ee147ff3c89f7fd5fc41ffc38055f3b Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 31 Oct 2022 21:31:08 +0800 Subject: [PATCH 020/112] =?UTF-8?q?=E5=88=A0=E9=99=A4=E5=A4=9A=E4=BD=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/fangjia/service/datastore/DistrictServiceImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictServiceImpl.java index 543a2f6ce..db0bfca4b 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictServiceImpl.java @@ -82,7 +82,6 @@ import com.fangjia.service.util.ServiceUtil; import com.fangjia.service.web.FangjiaHighlighter; import com.fangjia.service.web.MenuService; -import javax.jws.WebService; public class DistrictServiceImpl implements DistrictService { /** From 186bd022e94a74a42376871cbfa2a53fa8d6f23a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 31 Oct 2022 21:32:17 +0800 Subject: [PATCH 021/112] =?UTF-8?q?=E5=88=A0=E9=99=A4=E5=A4=9A=E4=BD=99?= =?UTF-8?q?=E5=BC=95=E5=85=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/service/map/AMapAddressServiceImpl.java | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/map/AMapAddressServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/map/AMapAddressServiceImpl.java index 13a75b35c..eec1baff6 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/map/AMapAddressServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/map/AMapAddressServiceImpl.java @@ -1,25 +1,12 @@ package com.fangjia.service.map; -import com.alibaba.fastjson.JSON; import com.fangjia.core.util.HttpRequest; -import com.fangjia.core.util.SourceNameUtil; -import com.fangjia.core.util.StringUtil; -import com.fangjia.dal.mongo.address.dao.AMapAddressDao; -import com.fangjia.dal.mongo.address.dao.AMapAddressDaoImpl; import com.fangjia.dal.mongo.address.model.AMapAddressPOIResult; import com.fangjia.service.bo.*; import com.fangjia.service.bo.map.*; -import com.fangjia.service.util.ABYun.ABuYunUtil; -import com.fangjia.service.util.AmapUtil; -import com.huaban.analysis.jieba.JiebaSegmenter; -import net.sf.ezmorph.array.AbstractArrayMorpher; -import net.sf.ezmorph.array.BooleanObjectArrayMorpher; -import net.sf.json.JSONArray; import net.sf.json.JSONObject; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; -import org.apache.lucene.util.CollectionUtil; -import org.omg.CORBA.OBJ_ADAPTER; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; From 9c4ade44cb0adc48350d43829dde0cb6d2fd050f Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 1 Nov 2022 13:42:26 +0800 Subject: [PATCH 022/112] =?UTF-8?q?=E5=AF=B9=E4=BA=8E=E8=BF=94=E5=9B=9E?= =?UTF-8?q?=E2=80=9Cnull=E2=80=9D=E5=88=A4=E6=96=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/weixin/WeiXinEvaluationController.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java index de0c0040b..38d005b33 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java @@ -283,7 +283,11 @@ public class WeiXinEvaluationController { evaluationAudit.setAuditStatus(3); evaluationAudit.setLastPrice(inquirePrice); evaluationAudit.setFeedbackStatus(feedbackStatus); - evaluationAudit.setFeedbackMsg((String)object.get("feedbackCallMsg")); + Object callMsg = object.get("feedbackCallMsg"); + if(callMsg!=null && !callMsg.toString().equals("null")){ + evaluationAudit.setFeedbackMsg(callMsg.toString()); + } + jsonObject.put("zsj",(int)(inquirePrice/ 10000));//终审价 bohaiEvaluationAuditService.updateBohaiEvaluationAudit(evaluationAudit); jsonObject.put("shzt", "审核完成"); From e2046d01e4142e2e4bdfc5fc15f2d673120f8f69 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 4 Nov 2022 11:35:19 +0800 Subject: [PATCH 023/112] =?UTF-8?q?=E7=94=B3=E8=AF=B7=E6=8A=A5=E5=91=8A?= =?UTF-8?q?=E8=B0=83=E7=94=A8=E7=99=BE=E7=9B=9B=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../weixin/WeiXinEvaluationController.java | 11 +++++++---- .../main/java/com/fangjia/open/util/BHBankUtils.java | 8 ++------ .../fangjia-open/src/main/webapp/static/js/report.js | 9 +++++++-- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java index 38d005b33..21121e8b7 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java @@ -1109,10 +1109,13 @@ public class WeiXinEvaluationController { formalReportService.update(formalReport); } } - Integer code = BHBankUtils.inquireToReport(bohaiEvaluationAudit.getQueryId(), model.getAddress()); - Map data = new HashMap(); - data.put("code", code); - jsonModel.setJsonModel(data); + JSONObject jsonObj = BHBankUtils.inquireToReport(bohaiEvaluationAudit.getQueryId(), model.getAddress()); + if(jsonObj!=null){ + Map data = new HashMap(); + data.put("code", (Integer)jsonObj.get("errCode")); + data.put("msg", (String)jsonObj.get("errMsg")); + jsonModel.setJsonModel(data); + } } } return jsonModel; diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java b/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java index 3ae49cc24..cda323214 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java @@ -431,8 +431,7 @@ public class BHBankUtils { } - - public static Integer inquireToReport(String queryId, String reportMsg){ + public static JSONObject inquireToReport(String queryId, String reportMsg){ String feedbackId = null; List list = new ArrayList(); Map params = new HashMap(); @@ -475,10 +474,7 @@ public class BHBankUtils { if(StringUtils.isNotBlank(content)){ JSONObject jsonObj = JSONObject.fromObject(content); if(jsonObj.containsKey("errCode")){ - Integer code = (Integer) jsonObj.get("errCode"); - if (code == 0) { - return code; - } + return jsonObj; } } }catch (Exception e){ diff --git a/projects/fangjia-open/src/main/webapp/static/js/report.js b/projects/fangjia-open/src/main/webapp/static/js/report.js index f3d833ccf..b089f29ff 100644 --- a/projects/fangjia-open/src/main/webapp/static/js/report.js +++ b/projects/fangjia-open/src/main/webapp/static/js/report.js @@ -375,8 +375,13 @@ function clickFormalReport(){ }, dataType:"json", success:function(data){ - $.cookie("fjt-formal-report",address, {path:"/",expires: 30}) - alert("鎮ㄧ殑鐢宠宸叉彁浜わ紝璇疯�愬績绛夊緟!"); + if(data && data.code!=0){ + alert(data.msg); + }else{ + $.cookie("fjt-formal-report",address, {path:"/",expires: 30}) + alert("鎮ㄧ殑鐢宠宸叉彁浜わ紝璇疯�愬績绛夊緟!"); + } + } }) } From 321baf13d86be1c13a90c904619a8f502d496e95 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sat, 5 Nov 2022 11:30:47 +0800 Subject: [PATCH 024/112] =?UTF-8?q?252=E5=9F=8E=E6=88=BF=E6=BA=90=E4=BF=AE?= =?UTF-8?q?=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index ca517f31a..8a8b30846 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -68,14 +68,14 @@ class BigdataForsaleBatch extends BaseBatch{ public void process(Object args) { def index = 0 def citys = CcbCommonUtils.getRunningCitys(args) - statCity(citys) def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ index++ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220918, 20221019); +// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220918, 20221019); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20221019); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); //测试通过匹配接口配房源效果 From f2a969b7e0cc113a66d2fa3303d360bae3ee4d3c Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sat, 5 Nov 2022 11:32:14 +0800 Subject: [PATCH 025/112] update --- projects/fangjia-coresite/pom.xml | 8 +++ .../src/main/webapp/header/v5/_footer.ftl | 4 +- .../ccbevaluation/ExportOracleData.groovy | 40 +++++++-------- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 10 ++-- .../fuzhou/DistrictPriceExportBatch.groovy | 49 ++++++++++++++----- 5 files changed, 73 insertions(+), 38 deletions(-) diff --git a/projects/fangjia-coresite/pom.xml b/projects/fangjia-coresite/pom.xml index 05fa7f275..72cefc4b9 100644 --- a/projects/fangjia-coresite/pom.xml +++ b/projects/fangjia-coresite/pom.xml @@ -242,5 +242,13 @@ 1.0.0 jdk18 + + + + + + + + diff --git a/projects/fangjia-coresite/src/main/webapp/header/v5/_footer.ftl b/projects/fangjia-coresite/src/main/webapp/header/v5/_footer.ftl index 323aeb051..b1a1c4956 100644 --- a/projects/fangjia-coresite/src/main/webapp/header/v5/_footer.ftl +++ b/projects/fangjia-coresite/src/main/webapp/header/v5/_footer.ftl @@ -38,7 +38,9 @@ 琛屼笟鍔ㄦ�� 鏁版嵁鍒嗘瀽 鎴愬姛妗堜緥 - 鍏徃鍔ㄦ�� + 鍚勮矾瀹� + 鎴夸环閫� + 鍦版暟 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index fed4c92bd..06e78b250 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -25,29 +25,29 @@ class ExportOracleData { def run(args) { citys = CcbCommonUtils.getRunningCitys(args) -// def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); + def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") def new_dists = [] //不会变 - exportEvaluationRateData() - exportBigdataPoiData()// - exportBigdataDistrictData()// +// exportEvaluationRateData() +// exportBigdataPoiData()// +// exportBigdataDistrictData()// +// +// exportGuidedPrice()//随人工改变// +// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策// +// +//// //处理小区情况 +// exportBigdataCommunity()// +// exportCommunityStaging()//没cityid +// exportCommunityBuileType()// - exportGuidedPrice()//随人工改变// - exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策// - -// //处理小区情况 - exportBigdataCommunity()// - exportCommunityStaging()//没cityid - exportCommunityBuileType()// - - exportThirdPartyPrice2(lastMonth, new_dists)// - exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 +// exportThirdPartyPrice2(lastMonth, new_dists)// +// exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 //第三方网站权重表 - exportWebsiteRate(); +// exportWebsiteRate(); //计算结果相关 @@ -63,11 +63,11 @@ class ExportOracleData { // exportCommunityPdescribeGj(curMonth) // exportCommCaseDetailGj(curMonth) -// def curDate = curMonth.toString()+"28" -// SSHHelper jschCommand = new SSHHelper(); -// System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}")); + def curDate = curMonth.toString()+"28" + SSHHelper jschCommand = new SSHHelper(); + System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}")); //ok文件 -// exportLastFile() + exportLastFile() } @@ -377,7 +377,7 @@ class ExportOracleData { "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + "from bigdata_deal_gj where CITYID='${cityCode}' and dealdate>=to_date('${dealDate}','yyyy-MM-dd')"; - allRows = dealData(sql) + allRows.addAll(dealData(sql)) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 6b1009e6c..0b40eb938 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202109//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202107 - def endDate = 202109 + def sDate= 202209//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202207 + def endDate = 202209 // def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") @@ -502,8 +502,8 @@ class ThirdPartyPriceBatch extends BaseBatch { def findThirdPartyPrice(def city, def statMap){ def q = new BasicDBObject() q.put("city", city) - - q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) + q.put("s_date", endDate) + // q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1)) def map = [:] for(def db : dbcursor){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy index 45f3cbe35..517bfef5c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy @@ -41,10 +41,18 @@ class DistrictPriceExportBatch extends BaseBatch{ def headers = ["id","comm_id","city","district","name","avg_price","year_month"] - def ret = createDatas("福州") - - saveToDB(ret) + def citys = CommonDatautils.findAllCity() + citys.remove("福州"); + citys.add(0, "福州") + def fz_datas = [] + for(def city : citys){ + def ret = createDatas("福州") + saveToDB(ret) + if(city == "福州"){ + fz_datas = ret + } + } def fileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".csv" def dataCodeFileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".sha512" @@ -52,7 +60,7 @@ class DistrictPriceExportBatch extends BaseBatch{ fileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".csv" dataCodeFileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".sha512" } - def sb = appendResult(headers, ret) + def sb = appendResult(headers, fz_datas) DataHandleUtils.outPutFile(fileName, sb, charset); FzFtpUtils.ftpUpload(fileName) //生成加密数据文件并上传 @@ -102,7 +110,12 @@ class DistrictPriceExportBatch extends BaseBatch{ // def districts = CommonDatautils.findAllDistricts(city, "district", null); // def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01"); // districts.addAll(removeDists) - def districts = getFzDistricts(city) + def districts = [] + if(city == "福州"){ + districts = getFzDistricts(city) + }else{ + districts = CommonDatautils.findAllDistricts(city, "district", null); + } def endDateStr = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-10" def dataStr = DateUtils.add(endDateStr, -30) @@ -127,14 +140,26 @@ class DistrictPriceExportBatch extends BaseBatch{ curPrice = dbo.get("avg_price") if(lastMonthPrice && curPrice && dbo.value_type!=1){ def markup = (curPrice - lastMonthPrice)*100/lastMonthPrice - if(Math.abs(markup)>3){ - def randMarkup = nextDouble(0, 3) - if(markup < 0){ - markup = randMarkup*(-1) - }else{ - markup = randMarkup + if(city == "福州"){ + if(Math.abs(markup)>3){ + def randMarkup = nextDouble(0, 3) + if(markup < 0){ + markup = randMarkup*(-1) + }else{ + markup = randMarkup + } + curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice) + } + }else{ + if(Math.abs(markup)>10){ + def randMarkup = nextDouble(0, 10) + if(markup < 0){ + markup = randMarkup*(-10) + }else{ + markup = randMarkup + } + curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice) } - curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice) } } }else{ From 668e5ec5dac21528a9bacda94c5d8c271f70c058 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Tue, 8 Nov 2022 17:26:16 +0800 Subject: [PATCH 026/112] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=9C=B0=E5=9D=80?= =?UTF-8?q?=E5=88=86=E7=B1=BB=E6=8E=A5=E5=8F=A3=E3=80=81=E6=B6=88=E8=B4=B9?= =?UTF-8?q?=E8=AF=84=E7=BA=A7=E3=80=81=E6=A5=BC=E7=9B=98=E8=AF=84=E7=BA=A7?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/AddressController.java | 163 +++++++++++++++++- .../model/CommunitySummaryRequestDto.java | 89 ++++++++++ 2 files changed, 247 insertions(+), 5 deletions(-) create mode 100644 projects/fangjia-open/src/main/java/com/fangjia/open/model/CommunitySummaryRequestDto.java diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index af890d880..2ae672241 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -20,6 +20,7 @@ import java.util.regex.Pattern; import javax.validation.Valid; import com.fangjia.core.util.*; +import com.fangjia.open.model.*; import net.sf.json.JSONArray; import net.sf.json.JSONObject; @@ -54,11 +55,6 @@ import com.fangjia.open.OpenConfig; import com.fangjia.open.dto.UniversalDto; import com.fangjia.open.interceptors.Authority; import com.fangjia.open.interceptors.Role; -import com.fangjia.open.model.AddesssMatchModel; -import com.fangjia.open.model.AddressModel; -import com.fangjia.open.model.BasicModel; -import com.fangjia.open.model.RateModel; -import com.fangjia.open.model.RestfulModelAndView; import com.fangjia.service.address.AddressParser; import com.fangjia.service.address.AddressParserService; import com.fangjia.service.address.AddressParserUtil; @@ -1931,6 +1927,163 @@ public class AddressController { return mav; } + /** + * 地址信息分类接口 + * @param model + * @return + */ + public Object communitySummaryAction(CommunitySummaryRequestDto model) { + + RestfulModelAndView mav = new RestfulModelAndView(); + if (StringUtils.isBlank(model.getAddress())) { + mav.setModel(BasicModel.buildBadRequestModel("地址不能为空")); + return mav; + } + + HttpRequest request = new HttpRequest(); + JSONObject json = null; + try{ + Map headers = new HashMap(); + headers.put("Content-Type", "application/json"); + + JSONObject reqBody = new JSONObject(); + reqBody.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity()); + reqBody.put("address", model.getAddress()); + reqBody.put("area", model.getArea()); + reqBody.put("maxFloor", model.getMaxFloor()); + reqBody.put("housePurpose", model.getHousePurpose()); + reqBody.put("memo", model.getMemo()); + + String result = request.postWithBody("http://192.168.0.100:32080/FangJiaAddressService/communitySummary", reqBody ,headers, "UTF-8"); + if(result != null){ + try{ + json = JSONObject.fromObject(result); + } catch(Exception e){ + json = null; + } + } + } catch(Exception e){ + e.printStackTrace(); + } + + if (json != null) { + int code = json.getInt("status"); + String msg = json.getString("msg"); + BasicModel basicModel = new BasicModel(code, msg); + if(json.containsKey("result")){ + basicModel.setResult(json.get("result")); + } + mav.setModel(basicModel); + } else { + mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); + } + return mav; + } + + /** + * 楼盘评级 + * @param model + * @return + */ + public Object buildingRatingAction(CommunitySummaryRequestDto model) { + + RestfulModelAndView mav = new RestfulModelAndView(); + if (StringUtils.isBlank(model.getAddress())) { + mav.setModel(BasicModel.buildBadRequestModel("地址不能为空")); + return mav; + } + + HttpRequest request = new HttpRequest(); + JSONObject json = null; + try{ + Map headers = new HashMap(); + headers.put("Content-Type", "application/json"); + + JSONObject reqBody = new JSONObject(); + reqBody.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity()); + reqBody.put("address", model.getAddress()); + reqBody.put("area", model.getArea()); + reqBody.put("maxFloor", model.getMaxFloor()); + reqBody.put("housePurpose", model.getHousePurpose()); + reqBody.put("memo", model.getMemo()); + String result = request.postWithBody("http://192.168.0.100:32080/FangJiaAddressService/buildingRating", reqBody, headers, "UTF-8"); + if(result != null){ + try{ + json = JSONObject.fromObject(result); + } catch(Exception e){ + json = null; + } + } + } catch(Exception e){ + e.printStackTrace(); + } + + if (json != null) { + int code = json.getInt("status"); + String msg = json.getString("msg"); + BasicModel basicModel = new BasicModel(code, msg); + if(json.containsKey("result")){ + basicModel.setResult(json.get("result")); + } + mav.setModel(basicModel); + } else { + mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); + } + return mav; + } + + /** + * 财力评级 + * @param model + * @return + */ + public Object financialRatingAction(CommunitySummaryRequestDto model) { + + RestfulModelAndView mav = new RestfulModelAndView(); + if (StringUtils.isBlank(model.getAddress())) { + mav.setModel(BasicModel.buildBadRequestModel("地址不能为空")); + return mav; + } + + HttpRequest request = new HttpRequest(); + JSONObject json = null; + try{ + Map headers = new HashMap(); + headers.put("Content-Type", "application/json"); + + JSONObject reqBody = new JSONObject(); + reqBody.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity()); + reqBody.put("address", model.getAddress()); + reqBody.put("area", model.getArea()); + reqBody.put("maxFloor", model.getMaxFloor()); + reqBody.put("housePurpose", model.getHousePurpose()); + reqBody.put("memo", model.getMemo()); + String result = request.postWithBody("http://192.168.0.100:32080/FangJiaAddressService/financialRating", reqBody, headers,"UTF-8"); + if(result != null){ + try{ + json = JSONObject.fromObject(result); + } catch(Exception e){ + json = null; + } + } + } catch(Exception e){ + e.printStackTrace(); + } + + if (json != null) { + int code = json.getInt("status"); + String msg = json.getString("msg"); + BasicModel basicModel = new BasicModel(code, msg); + if(json.containsKey("result")){ + basicModel.setResult(json.get("result")); + } + mav.setModel(basicModel); + } else { + mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); + } + return mav; + } + public Object similarityAction(AddesssMatchModel model) { RestfulModelAndView mav = new RestfulModelAndView(); if (StringUtils.isBlank(model.getAddress_1()) || StringUtils.isBlank(model.getAddress_2())) { diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/CommunitySummaryRequestDto.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/CommunitySummaryRequestDto.java new file mode 100644 index 000000000..6c5cdd50d --- /dev/null +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/CommunitySummaryRequestDto.java @@ -0,0 +1,89 @@ +package com.fangjia.open.model; + + +/** + * 灏忓尯瀹借〃鍏ュ弬DTO + * + * @author shenjian + * @since 2022/10/11 + */ +public class CommunitySummaryRequestDto extends FangjiaApiModel { + + /** + * 鍩庡競 + */ + private String city; + + /** + * 鍦板潃 + */ + private String address; + + /** + * 鎬绘ゼ灞� + */ + private Integer maxFloor; + + /** + * 闈㈢Н + */ + private Float area; + + /** + * 鎴垮眿鐢ㄩ�� + */ + private String housePurpose; + + /** + * 澶囨敞 + */ + private String memo; + + public String getCity() { + return city; + } + + public void setCity(String city) { + this.city = city; + } + + public String getAddress() { + return address; + } + + public void setAddress(String address) { + this.address = address; + } + + public Integer getMaxFloor() { + return maxFloor; + } + + public void setMaxFloor(Integer maxFloor) { + this.maxFloor = maxFloor; + } + + public Float getArea() { + return area; + } + + public void setArea(Float area) { + this.area = area; + } + + public String getHousePurpose() { + return housePurpose; + } + + public void setHousePurpose(String housePurpose) { + this.housePurpose = housePurpose; + } + + public String getMemo() { + return memo; + } + + public void setMemo(String memo) { + this.memo = memo; + } +} From 8a652a38b71c9ed6a860fbbbe88cff5e14af5bbe Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Wed, 9 Nov 2022 09:26:47 +0800 Subject: [PATCH 027/112] =?UTF-8?q?=E7=BC=96=E7=A0=81=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E4=B8=BAgbk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/model/CommunitySummaryRequestDto.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/CommunitySummaryRequestDto.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/CommunitySummaryRequestDto.java index 6c5cdd50d..c5af8f307 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/CommunitySummaryRequestDto.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/CommunitySummaryRequestDto.java @@ -2,7 +2,7 @@ package com.fangjia.open.model; /** - * 灏忓尯瀹借〃鍏ュ弬DTO + * 小区宽表入参DTO * * @author shenjian * @since 2022/10/11 @@ -10,32 +10,32 @@ package com.fangjia.open.model; public class CommunitySummaryRequestDto extends FangjiaApiModel { /** - * 鍩庡競 + * 城市 */ private String city; /** - * 鍦板潃 + * 地址 */ private String address; /** - * 鎬绘ゼ灞� + * 总楼层 */ private Integer maxFloor; /** - * 闈㈢Н + * 面积 */ private Float area; /** - * 鎴垮眿鐢ㄩ�� + * 房屋用途 */ private String housePurpose; /** - * 澶囨敞 + * 备注 */ private String memo; From b734dfc285522ab8f4c022ec3988fb945762f79f Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 9 Nov 2022 13:14:51 +0800 Subject: [PATCH 028/112] update --- .../ccbevaluation/BigdataForsaleBatch.groovy | 16 +-- .../ccbevaluation/ExportOracleDataTest.groovy | 8 +- .../ccbevaluation/GuiDingPriceBatch.groovy | 2 +- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 9 +- .../ccbevaluation/data/AddressCut.groovy | 97 ++++++++++++++++++- .../ccbevaluation/data/CcbDataStat.groovy | 18 +++- .../script/gj/es/EsMonthHouseStat.groovy | 4 +- 7 files changed, 135 insertions(+), 19 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 5b0edbae1..53c7c83c8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -67,7 +67,7 @@ class BigdataForsaleBatch extends BaseBatch{ @Override public void process(Object args) { def index = 0 - def citys = CcbCommonUtils.getRunningCitys(args) + def citys = ["阿坝州"]//CcbCommonUtils.getRunningCitys(args) statCity(citys) def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ @@ -75,8 +75,8 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220918, 20221019); -// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011); +// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220918, 20221019); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221019); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); //测试通过匹配接口配房源效果 // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118) @@ -477,8 +477,14 @@ class BigdataForsaleBatch extends BaseBatch{ println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type continue; } + + def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room) + if(vallaFlag){ + continue; + } } + if(!seawater.total_price){ continue; } @@ -538,10 +544,6 @@ class BigdataForsaleBatch extends BaseBatch{ if(floor && floor.toInteger()<1){ floor = null } - def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room) - if(vallaFlag){ - continue; - } def rowMap = [:]; if(cat == "rent"){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy index 431fc02c2..c4d5579f4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy @@ -976,9 +976,9 @@ class ExportOracleDataTest { }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" - def fileName = "/ccb_evaluation/MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat" } def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); @@ -1007,9 +1007,9 @@ class ExportOracleDataTest { }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" - def fileName = "/ccb_evaluation/MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat" + def fileName = "/ccb_evaluation/MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat" } def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 6981c7149..bf34601ed 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2022-10-22", "2022-10-27") + def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-09") if(prices){ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 6b1009e6c..02afd29da 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202109//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202107 - def endDate = 202109 + def sDate= 202204//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202202 + def endDate = 202204 // def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") @@ -503,7 +503,10 @@ class ThirdPartyPriceBatch extends BaseBatch { def q = new BasicDBObject() q.put("city", city) + //q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) + q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) + def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1)) def map = [:] for(def db : dbcursor){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy index 19bf6561b..4c66509ec 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/AddressCut.groovy @@ -7,8 +7,12 @@ import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.lixue.util.DBUtils import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.elasticsearch.xpack.sql.jdbc.EsDataSource +import java.sql.ResultSet import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService @@ -16,13 +20,79 @@ import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong class AddressCut extends BaseBatch{ + + + final DBCollection thirdPartyPriceColl = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018) + + public static EsDataSource dataSource = new EsDataSource() + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000" + static { + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + connectionProperties.put("user", "developer") + connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") + dataSource.setProperties(connectionProperties) + } static main(args) { runBatch(new AddressCut(), args); } @Override public void process(Object args) { - sameAddr() +// sameAddr() //相同地址小区 +// roadMatch() + addAnjukeRoad() + + } + + def addAnjukeRoad(){ + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区地址拆分\\小区地址切分(结果)_1.xlsx", { d -> + def roads = findThirdPartyPrice(d['A'], d['B'], d['C'], "链家") + if(roads){ + d['O'] = roads[0] + } + }); + } + + def findThirdPartyPrice(city, region, name, source){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put("source", source) + def dbcursor = thirdPartyPriceColl.find(q).sort(new BasicDBObject("s_date", -1)).limit(10) + def roads = [] + for(def d : dbcursor){ + if(d.address && !roads.contains(d.address)){ + roads << d.address + } + } + return roads + } + + def roadMatch(){ + def cityRoadsMap = queryPoiMap() + println "道路加载完成....." + ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区地址拆分\\小区地址切分(结果)_1.xlsx", { d -> + def roads = cityRoadsMap.get(d['A']) +// if (roads && roads.contains(d['E'])) { +// d['L'] = "是" +// println d['A'] + " " + d['E'] +// } + + if(roads){ + for(def road : roads){ + if(!d['L'] && d['E'] && d['E'].toString().contains(road)){ + d['L'] = "中间包含" + d['M'] = road + println d['A'] + " " + d['E'] + break; + } + } + } + }) + + } def sameAddr() { @@ -143,4 +213,29 @@ class AddressCut extends BaseBatch{ latch.await() return rows; } + + def queryPoiMap(){ + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("SELECT cityname,name,type FROM amap_poi WHERE 1=1") +// sqlBuffer.append(" and cityname = '" + city + "'") +// sqlBuffer.append(" and name = '" + name + "'") + sqlBuffer.append(" and type like '%道路名%'") +// sqlBuffer.append(" and MATCH(name,'${name}')") +// sqlBuffer.append(" and source = '高德' order by score() desc") + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + def cityMap = [:] + while (rs.next()) { + def city = rs.getString("cityname"); + def road = rs.getString("name") + def roads = cityMap.get(city) + if(!roads){ + roads = new HashSet() + } + roads << road + cityMap.put(city, roads) + + } + return cityMap; + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDataStat.groovy index 1aa5a391f..d7be2d646 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDataStat.groovy @@ -1,6 +1,7 @@ package com.fangjia.script.ccbevaluation.data import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils @@ -19,7 +20,22 @@ class CcbDataStat { static main(args) { def task = new CcbDataStat() // task.run() - task.tradeRun() +// task.tradeRun() + task.regionDistStat() + } + + def regionDistStat(){ + def citys = CcbCommonUtils.citys_252 + def rows = [] + for(def c : citys){ + def regions = CommonDatautils.getRegionListByCity(c) + for(def r : regions){ + def count = CommonDatautils.findRegionDistrictCount("district",c, r) + rows << [c, r, count] + println c +" "+r +" "+count + } + } + ExcelUtils.write("d:\\数据\\建行\\区域小区量统计.xlsx", rows, "统计") } def tradeRun(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy index 3d79e39fb..9c68829cd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy @@ -19,11 +19,11 @@ class EsMonthHouseStat { def task = new EsMonthHouseStat() // task.run() // task.tradeRun() -// task.houseRun() + task.houseRun() // task.statHouseUpdateCount() - task.statDistHouse() +// task.statDistHouse() } def statDistHouse(){ From 477950d1e73c4b21894b9cb17c459fd3f4602385 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 9 Nov 2022 15:22:36 +0800 Subject: [PATCH 029/112] =?UTF-8?q?=E8=9E=8D=E5=90=88=E5=9F=8E=E5=B8=82?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy index 77d788a46..c5b66e238 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy @@ -42,7 +42,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { @Override public void process(Object args) { - def citys = CcbCommonUtils.citys_all; + def citys = CcbCommonUtils.getRunningCitys(args) def rows = [] def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() // for (def city : citys) { From cd2790fcd005e7e9aa8bebbe7f4f967324d82dfa Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 9 Nov 2022 17:21:57 +0800 Subject: [PATCH 030/112] =?UTF-8?q?=E7=BC=96=E7=A0=81=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ronghe/ExportDistrictRongHe.groovy | 76 +++++++++---------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 386f96f4b..64580b128 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -30,7 +30,7 @@ class ExportDistrictRongHe extends BaseBatch{ citys = CcbCommonUtils.getRunningCitys(args) def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] - //鎸夊ぉ瀵硷紙姣忓ぉ涓�涓枃浠跺す锛� + //按天导(每天一个文件夹) exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) if(curDate.toString().endsWith("10")){ @@ -63,14 +63,14 @@ class ExportDistrictRongHe extends BaseBatch{ // def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() // // -//// //妫�鏌k鏂囦欢鏄惁涓婁紶 +//// //检查ok文件是否上传 //// exportAllBigdataDealFJW() //// for(def city : citys) { //// def cityCode = cityCodeDataMap.get(city) //// -// //鎸夊ぉ瀵�(涓�涓枃浠跺す涓�) +// //按天导(一个文件夹下) // //exportBigdataDealFJW(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd")) -// //鎸夊ぉ瀵硷紙姣忓ぉ涓�涓枃浠跺す锛� +// //按天导(每天一个文件夹) // exportBigdataDealFJW2(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd")) // //// exportBigdataCommunityGJRH(cityCode) @@ -103,7 +103,7 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" } def index = 0 for(def city : citys){ @@ -145,9 +145,9 @@ class ExportDistrictRongHe extends BaseBatch{ } } - println "BIGDATA_COMMUNITY_FJW 鏁版嵁鏂囦欢杈撳嚭瀹屾垚......" + println "BIGDATA_COMMUNITY_FJW 数据文件输出完成......" CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_COMMUNITY_FJW 涓婁紶Ftp瀹屾垚......" + println "BIGDATA_COMMUNITY_FJW 上传Ftp完成......" } def exportBigdataCommbuildstruGJ(){ def allRows = [] @@ -170,11 +170,11 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_COMMBUILDSTRU_FJW 涓婁紶Ftp瀹屾垚......" + println "BIGDATA_COMMBUILDSTRU_FJW 上传Ftp完成......" } def exportBigdataCommhousetypeGJ(){ @@ -199,11 +199,11 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_COMMHOUSETYPE_FJW 涓婁紶Ftp瀹屾垚......" + println "BIGDATA_COMMHOUSETYPE_FJW 上传Ftp完成......" } def exportBigdataCommbuildyearGj(){ @@ -227,11 +227,11 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh", fileName, curDate) - println "BIGDATA_COMMBUILDYEAR_FJW 涓婁紶Ftp瀹屾垚......" + println "BIGDATA_COMMBUILDYEAR_FJW 上传Ftp完成......" } def exportBigdataCommfeeGj(){ @@ -265,11 +265,11 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_COMMFEE_FJW 涓婁紶Ftp瀹屾垚......" + println "BIGDATA_COMMFEE_FJW 上传Ftp完成......" } @@ -295,11 +295,11 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_COMMHOUSEUSE_FJW 涓婁紶Ftp瀹屾垚......" + println "BIGDATA_COMMHOUSEUSE_FJW 上传Ftp完成......" } def exportBigdataPositionGJ(){ @@ -328,11 +328,11 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_POSITION_FJW 涓婁紶Ftp瀹屾垚......" + println "BIGDATA_POSITION_FJW 上传Ftp完成......" } def exportBigdataDistrictFJW(){ @@ -360,11 +360,11 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh", fileName, curDate) - println "BIGDATA_DISTRICT_FJW 涓婁紶Ftp瀹屾垚......" + println "BIGDATA_DISTRICT_FJW 上传Ftp完成......" } def exportBigdataAliasFJW(){ @@ -395,11 +395,11 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "ADDRESS_ALIAS_FJW 涓婁紶Ftp瀹屾垚......" + println "ADDRESS_ALIAS_FJW 上传Ftp完成......" } def exportBigdataDealFJW(dealdate){ @@ -419,7 +419,7 @@ class ExportDistrictRongHe extends BaseBatch{ def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" } def allRows = [] @@ -504,7 +504,7 @@ class ExportDistrictRongHe extends BaseBatch{ } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh", fileName, curDate) - println "BIGDATA_DEAL_FJW 涓婁紶Ftp瀹屾垚......${curStartStr}" + println "BIGDATA_DEAL_FJW 上传Ftp完成......${curStartStr}" def curEndStr = DateUtils.add(curStartStr, 1) curStartDateStr = curEndStr } @@ -519,7 +519,7 @@ class ExportDistrictRongHe extends BaseBatch{ def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" } def allRows = [] @@ -548,7 +548,7 @@ class ExportDistrictRongHe extends BaseBatch{ FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_DEAL_FJW 涓婁紶Ftp瀹屾垚......${endDateStr}" + println "BIGDATA_DEAL_FJW 上传Ftp完成......${endDateStr}" } def createDeal(sql){ @@ -709,10 +709,10 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); - println "BIGDATA_DEAL_FJW 瀵煎嚭瀹屾垚......" + println "BIGDATA_DEAL_FJW 导出完成......" } @@ -720,7 +720,7 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" } def index = 0 for(def city : citys) { @@ -770,7 +770,7 @@ class ExportDistrictRongHe extends BaseBatch{ } CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_BUILDING_FJW 涓婁紶Ftp瀹屾垚......" + println "BIGDATA_BUILDING_FJW 上传Ftp完成......" } @@ -794,40 +794,40 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "EVALUATE_RATE_FJW 涓婁紶Ftp瀹屾垚......" + println "EVALUATE_RATE_FJW 上传Ftp完成......" } def exportBigdataBuildprojFjw(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile([], fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_BUILDPROJ_FJW 涓婁紶Ftp瀹屾垚......" + println "BIGDATA_BUILDPROJ_FJW 上传Ftp完成......" } def exportBigdataCommlayoutpicFjw(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile([], fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_COMMLAYOUTPIC_FJW 涓婁紶Ftp瀹屾垚......" + println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......" } def exportOk(curDate){ // def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok" if(CommonDatautils.isWindow()){ - dataCodeFileName = "D:\\鏁版嵁\\寤鸿\\鎴夸骇浼板�糪\鏁版嵁鏂囦欢\\txt\\ccb_rh\\${curDate}\\${curDate}.ok" + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok" } FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); CcbFtpUtils.ftpUpload("rh",dataCodeFileName, curDate) From 129050ddc4eb971e3a95c8c07960c996d3bf0195 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 10 Nov 2022 11:50:59 +0800 Subject: [PATCH 031/112] =?UTF-8?q?=E5=AF=B9=E7=99=BE=E7=9B=9B=E4=B8=8A?= =?UTF-8?q?=E4=BC=A0=E5=9B=BE=E7=89=87=E8=BF=94=E5=9B=9E=E8=BF=9B=E8=A1=8C?= =?UTF-8?q?=E6=8E=A7=E5=88=B6=EF=BC=8C=E9=98=B2=E6=AD=A2=E4=B8=8A=E4=BC=A0?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/fangjia/core/util/DataUtil.java | 2 +- .../weixin/WeiXinEvaluationController.java | 7 ++++++- .../java/com/fangjia/open/util/BHBankUtils.java | 7 ++----- .../main/webapp/weiXinEvaluation/bhreport.ftl | 17 ++--------------- 4 files changed, 11 insertions(+), 22 deletions(-) diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java b/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java index 4aaabb88f..608508eaa 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java @@ -1845,7 +1845,7 @@ public class DataUtil { // // System.out.println(FourUpSixInto("258.393", 2)); - String href = encodePicUrl("62afd81d303c3f827833bb03", "o", "upload"); + String href = encodePicUrl("636b079c303c3f7093fd952a", "o", "upload"); String link = "http://static.fangjia.com/stc/fs/"+href; System.out.println(link); } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java index 21121e8b7..ed8766b8c 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java @@ -1364,7 +1364,12 @@ public class WeiXinEvaluationController { System.out.println(link); } if(uploadfiles.size()>0){ - BHBankUtils.inquireUploadFiles2(usersDetail.getName(),model.getQueryId(), model.getFileClass(), uploadfiles); + JSONObject jsonObj = BHBankUtils.inquireUploadFiles2(usersDetail.getName(),model.getQueryId(), model.getFileClass(), uploadfiles); + if(jsonObj!=null && (Integer)jsonObj.get("errCode")!=0) { + mav.setModel(BasicModel.buildServerErrorModel().setMsg((String) jsonObj.get("errMsg"))); + }else{ + mav.setModel(BasicModel.buildServerErrorModel().setMsg(("文件上传成功!"))); + } } } catch (Exception e) { e.printStackTrace(); diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java b/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java index cda323214..6a861a303 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java @@ -349,7 +349,7 @@ public class BHBankUtils { } - public static Integer inquireUploadFiles2(String name, String queryId, Integer fileClass , List uploadfiles){ + public static JSONObject inquireUploadFiles2(String name, String queryId, Integer fileClass , List uploadfiles){ Map params = new HashMap(); params.put("queryId",queryId); params.put("userName",name); @@ -420,10 +420,7 @@ public class BHBankUtils { String content = response.body().string(); JSONObject jsonObj = JSONObject.fromObject(content); System.out.println("上传接口返回:"+jsonObj.toString()); - Integer code = (Integer) jsonObj.get("errCode"); - if(code == 0){ - return code; - } + return jsonObj; }catch (Exception e){ e.printStackTrace(); } diff --git a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl index eaab26cb0..ecf07ff92 100644 --- a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl +++ b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl @@ -499,23 +499,10 @@ contentType: false, processData: false, success: function (data) { - if(data == null){ + + if(data.msg && data.msg.indexOf("鎴愬姛")>-1){ alert("涓婁紶鎴愬姛"); window.location.reload(); - // $("#file_wrap_list").remove(); - // var boxes = $("input[name=reportFiles]") - // if(boxes.length>1){ - // var i = 0; - // boxes.forEach(function(element) { - // i++; - // if(i>2){ - // element.remove(); - // } - // }); - // } - // $("a.create-actions08").empty().text("璇烽�夋嫨绫诲瀷") - // $("#fileClass").val("") - // $("a.create-actions08").removeClass("selected") }else{ alert(data.msg); } From c3ad317dfa2f3a44a48523a6893797fbbdbc9d0a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 11 Nov 2022 10:35:35 +0800 Subject: [PATCH 032/112] update --- .../fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 1caff80e1..a84057ac1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -41,7 +41,7 @@ class SynchDistrictPrice extends BaseBatch{ println "starting......" def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd") - def citys = ["福州"]//CommonUtils.findAllCity() + def citys = CommonUtils.findAllCity() esSources.each{esSource, mgSource-> for(def city : citys){ From 1b22b834437c2a092e1687e853c0b015ae182d5b Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 11 Nov 2022 11:43:39 +0800 Subject: [PATCH 033/112] =?UTF-8?q?=E8=A1=8C=E6=83=85=E5=B0=8F=E5=8C=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../BigdataCommunityBatch.groovy | 2 +- .../ccbevaluation/BigdataForsaleBatch.groovy | 1 - .../ccbevaluation/CcbCommonUtils.groovy | 2 +- .../fangjia/script/ccbevaluation/Test.groovy | 44 +- .../ronghe/BigdataCitypriceBatch.groovy | 582 ++++++++++++++++++ .../ronghe/ExportDistrictRongHe.groovy | 2 +- .../script/fuzhou/FzDataResultStat.groovy | 2 +- .../com/fangjia/script/fuzhou/Test1.groovy | 2 +- .../com/fangjia/script/util/DataUtil.groovy | 5 +- 9 files changed, 614 insertions(+), 28 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 2439e06d8..b9a415811 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -28,7 +28,7 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataCommunityBatch extends BaseBatch{ - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) static main(args) { runBatch(new BigdataCommunityBatch(), args); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 27732efd9..73b4cf4b0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -68,7 +68,6 @@ class BigdataForsaleBatch extends BaseBatch{ public void process(Object args) { def index = 0 def citys = CcbCommonUtils.getRunningCitys(args) - def citys = ["阿坝州"]//CcbCommonUtils.getRunningCitys(args) def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ index++ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 46aa81fac..d9d7660c0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -16,7 +16,7 @@ import groovy.sql.Sql class CcbCommonUtils { // - public static def citys_all = [ "长春","上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"] + public static def citys_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"] public static def citys_92 = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"] public static def citys_252 = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index e987ea5af..6326f35f4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -446,6 +446,10 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business //// } ////},0) // + +ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d-> +},0) + // // // @@ -1001,30 +1005,30 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business -def fileName = "D:\\数据\\建行\\房产估值\\指导价\\人工指导价录入_1020.xlsx" - -ExcelUtils.multiThreadingModify(fileName, {d-> - if(d['A']){ -// def region = RegionData.formatRegion(d['A'], d['B']) -// if(region){ -// def dist = CommonDatautils.findDistrict(d['A'], region, d['C'], "district") +//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\人工指导价录入_1020.xlsx" +// +//ExcelUtils.multiThreadingModify(fileName, {d-> +// if(d['A']){ +//// def region = RegionData.formatRegion(d['A'], d['B']) +//// if(region){ +//// def dist = CommonDatautils.findDistrict(d['A'], region, d['C'], "district") +//// if(dist){ +//// d['E'] = dist.region +//// d['F'] = dist._id.toString() +//// } +//// } +// if(!d['F']){ +// def dist = CommonDatautils.findDistrictById(d['A']) +//// // if(dist){ -// d['E'] = dist.region -// d['F'] = dist._id.toString() +//// d['H'] = dist.region +//// d['G'] = dist.name +// d['F'] = dist.status // } // } - if(!d['F']){ - def dist = CommonDatautils.findDistrictById(d['A']) // - if(dist){ -// d['H'] = dist.region -// d['G'] = dist.name - d['F'] = dist.status - } - } - - } -},0) +// } +//},0) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index 1656648bd..638323257 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -1,15 +1,597 @@ package com.fangjia.script.ccbevaluation.ronghe import com.fangjia.script.BaseBatch +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import org.apache.commons.lang.StringUtils +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong class BigdataCitypriceBatch extends BaseBatch{ + public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"]; + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + static main(args) { runBatch(new BigdataCitypriceBatch(), args) } @Override void process(Object args) { + def citys = CcbCommonUtils.getRunningCitys(args) + def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger(); + def endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger(); + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +// cityRun(citys, startDate, endDate, cityCodeDataMap) +// regionRun(citys, startDate, endDate, cityCodeDataMap) + districtRun(citys, startDate, endDate, cityCodeDataMap) } + + def cityRun(citys, startDate, endDate, cityCodeDataMap){ + def rets = [] + def index = 0 + for(def city : citys){ + index ++ + def row = createDatas(city, null, startDate, endDate, cityCodeDataMap, null) + if(row){ + rets << row + } + println "[ ${index} ]"+city+" 完成 " + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "10"; + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_CITYPRICE.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_CITYPRICE.dat" + } + def headers = [] + FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false); + println "BIGDATA_CITYPRICE 数据文件输出完成......" + } + + + def regionRun(citys, startDate, endDate, cityCodeDataMap){ + def regionDatas = [] + for(def c : citys) { + def regionMap = CcbCommonUtils.findAllDistrictCodeValue(c) + regionMap.each{region, regionCodeDataMap-> + regionDatas << [c, region, regionCodeDataMap] + } + } + + def rets = new CopyOnWriteArrayList() + def size = regionDatas.size() + final CountDownLatch latch = new CountDownLatch(size) + def totalCount = new AtomicLong() + for(def i = 0;i3){ + sell_avg_price = avgValue(sellMap.avg_price) + sell_avg_area = avgValue(sellMap.area) + sell_avg_total = avgValue(sellMap.total_price) + }else{ + sell_count = 0 + } + } + + def rent_avg_price = null + def rent_avg_area = null + def rent_avg_total = null + def rentStartDate = null + def rent_count = 0 + def rentMap = findHouses(dist.city, dist.region, "rent", startDate, endDate, dist.name, dist.alias, dist.estate_type2) + if(!rentMap || (rentMap && rentMap.avg_price.size()<3)){ + rentDataType = 3 + rentStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -3), "yyyyMMdd").toInteger(); + rentMap = findHouses(dist.city, dist.region, "rent", startDate, endDate, dist.name, dist.alias, dist.estate_type2) + if(!rentMap || (rentMap && rentMap.avg_price.size()<3)){ + rentDataType = 6 + rentStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -6), "yyyyMMdd").toInteger(); + rentMap = findHouses(dist.city, dist.region, "rent", startDate, endDate, dist.name, dist.alias, dist.estate_type2) + } + } + if(rentMap){ + rent_count = rentMap.avg_price.size() + if(rent_count>3){ + rent_avg_price = avgValue(rentMap.avg_price) + rent_avg_area = avgValue(rentMap.area) + rent_avg_total = avgValue(rentMap.total_price) + }else{ + rent_count = 0 + } + } + if (sell_avg_price || rent_avg_price) { + def row = [] + row << cityCodeDataMap.get(dist.city) + row << DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger(); + row << dist._id.toString() + + if (sell_avg_price) { + row << sell_avg_price.toInteger() + } else { + row << '' + } + row << sell_count + + if (rent_avg_price) { + row << DataHandleUtils.getDoubleRound(2, rent_avg_price) + } else { + row << '' + } + row << rent_count + + row << DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + + if (sell_avg_area) { + row << DataHandleUtils.getDoubleRound(2, sell_avg_area) + } else { + row << '' + } + + if (rent_avg_area) { + row << DataHandleUtils.getDoubleRound(2, rent_avg_area) + } else { + row << '' + } + + if (sell_avg_total) { + row << DataHandleUtils.getDoubleRound(2, sell_avg_total) + } else { + row << '' + } + + if (rent_avg_total) { + row << DataHandleUtils.getDoubleRound(2, rent_avg_total) + } else { + row << '' + } + row << '11' + row << sellDataType + row << rentDataType + row << 1 + row << 'FJW' + return row + } + return null + } + + + def createDatas(city, region , startDate, endDate, cityCodeDataMap, regionCodeDataMap) { + def sellDataType = 1 + def rentDataType = 1 + def sell_count = findHouseCountByType(city, region, "sell", startDate, endDate) + def sell_avg_price = null + def sell_avg_area = null + def sell_avg_total = null + def sellStartDate = null + if (sell_count < 10) { + sellDataType = 3 + sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -3), "yyyyMMdd").toInteger(); + sell_count = findHouseCountByType(city, region, "sell", sellStartDate, endDate) + if (sell_count < 10) { + sellDataType = 6 + sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -6), "yyyyMMdd").toInteger(); + sell_count = findHouseCountByType(city, region, "sell", sellStartDate, endDate) + } + } else { + sellStartDate = startDate + } + if (sell_count) { + sell_avg_price = findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"avg_price") + sell_avg_area = findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"area") + sell_avg_total = findHouseAvgByType(city, region, "sell", sellStartDate, endDate, "total_price") + } + + + def rentStartDate = null + def rent_count = findHouseCountByType(city, region, "rent", startDate, endDate) + if (rent_count < 10) { + rentDataType = 3 + rentStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -3), "yyyyMMdd").toInteger(); + rent_count = findHouseCountByType(city, region, "sell", startDate, endDate) + if (rent_count < 10) { + rentDataType = 6 + rentStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -6), "yyyyMMdd").toInteger(); + rent_count = findHouseCountByType(city, region, "sell", rentStartDate, endDate) + } + } else { + rentStartDate = startDate + } + + def rent_avg_price = null + def rent_avg_area = null + def rent_avg_total = null + if (rent_count>=10) { + rent_avg_price = findHouseAvgByType(city, region, "rent", rentStartDate, endDate, "avg_price") + rent_avg_area = findHouseAvgByType(city, region, "rent", rentStartDate, endDate, "area") + rent_avg_total = findHouseAvgByType(city, region, "rent", rentStartDate, endDate, "total_price") + } + + println city +" "+region +" sell: "+ sell_count +" rent: "+rent_count + + if (sell_avg_price || rent_avg_price) { + def row = [] + row << cityCodeDataMap.get(city) + row << DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger(); + + if(region){ + row << regionCodeDataMap.district_code + } + + if (sell_avg_price) { + row << sell_avg_price.toInteger() + } else { + row << '' + } + + if (rent_avg_price) { + row << DataHandleUtils.getDoubleRound(2, rent_avg_price) + } else { + row << '' + } + + row << sell_count + row << rent_count + row << DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + row << '11' + + if (sell_avg_area) { + row << DataHandleUtils.getDoubleRound(2, sell_avg_area) + } else { + row << '' + } + + if (rent_avg_area) { + row << DataHandleUtils.getDoubleRound(2, rent_avg_area) + } else { + row << '' + } + + if (sell_avg_total) { + row << DataHandleUtils.getDoubleRound(2, sell_avg_total) + } else { + row << '' + } + + if (rent_avg_total) { + row << DataHandleUtils.getDoubleRound(2, rent_avg_total) + } else { + row << '' + } + + row << 1 + row << sellDataType + row << rentDataType + row << 'FJW' + return row + } + + return null + } + + + def avgValue(def datas){ + def avgValue = null + if(datas){ + avgValue = datas.sum()/datas.size() + } + return avgValue; + } + + def findHouseAvgByType(def city, def region, def cat, def startDate, def endDate, def type){ + String sql = "select avg(${type}) as avg_val from latest_seawater where cat='${cat}' and city='${city}' and p_date >= ${startDate} and p_date<=${endDate} " + if(region){ + sql = sql +" and region='${region}'" + } + def val = null + ElasticUtils.executeSqlQuery(sql, { d -> + val = d.avg_val + }) + return val; + } + + def findHouseCountByType(def city, def region, def cat, def startDate, def endDate){ + String sql = "select count(1) as count from latest_seawater where cat='${cat}' and city='${city}' and p_date >= ${startDate} and p_date<=${endDate} " + if(region){ + sql = sql +" and region='${region}'" + } + def val = null + ElasticUtils.executeSqlQuery(sql, { d -> + val = d.count + }) + return val; + } + + def findHouseDetail(def city, def region, def cat, def startDate, def endDate, def estate_type2){ + + String sql = "select estate_type,area,avg_price,total_price,name,height,room c from latest_seawater where cat='${cat}' and city='${city}' and p_date >= ${startDate} and p_date<=${endDate} " + if(region){ + sql = sql +" and region='${region}'" + } + def avgPrices = [] + def areas = [] + def totalPrices = [] + ElasticUtils.executeSqlQuery(sql,{ seawater-> + def flag = true + if(estate_type2 && estate_type2=="普通住宅"){ + if(seawater.estate_type && (seawater.estate_type.contains("商住") || seawater.estate_type.contains("商业") || seawater.estate_type.contains("商务"))){ + flag = false + } + + if(seawater.estate_type&& seawater.estate_type.contains("别墅")){ + flag = false + } + + def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room) + if(vallaFlag){ + flag = false + } + } + if(cat == "rent"){ + if(seawater.avg_price && seawater.avg_price>1000){ + flag = false + } + + if(seawater.avg_price && seawater.avg_price<10){ + flag = false + } + } + if(flag){ + avgPrices << seawater.avg_price + areas << seawater.area + totalPrices << seawater.total_price + } + }) + def retMap = [:] + if(areas){ + retMap.avg_price = avgPrices + retMap.area = areas + retMap.total_price = totalPrices + } + return retMap; + } + + + + def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(true); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) + .must(QueryBuilders.termQuery("cat", cat)).must(date_range).must(area_range) + + if(region){ + queryBuilder.must(QueryBuilders.termQuery("region", region)) + } + + if(name || alias){ + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + + if(name.contains("弄小区")){ + def newName = name.replace("弄小区","弄") + if(!alias.contains(newName)){ + alias << newName + } + } + + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + } + + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + + def avgPrices = [] + def areas = [] + def totalPrices = [] + + if(houses){ + for (int k=0;k1000){ + continue + } + + if(seawater.avg_price && seawater.avg_price<10){ + continue + } + } + avgPrices << seawater.avg_price + areas << seawater.area + totalPrices << seawater.total_price + } + } + + def retMap = [:] + if(areas){ + retMap.avg_price = avgPrices + retMap.area = areas + retMap.total_price = totalPrices + } + return retMap; + } + + /** + * 是否是别墅 + * @return + */ + public boolean isVilla(area, name, height, room) { + if (area == null) { + return false; + } + if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) { + return true; + } + + boolean ret = false; + if (height != null && room != null) { + if (height <= 3) { + if (area > 200 && room >= 5) { + ret = true; + } + } + } else if (height == null && room != null) { + if (area > 360 && room >= 6) { + ret = true; + } + } else if (height == null && room == null) { + if (area > 500) { + ret = true; + } + } + + return ret; + } + + + + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 64580b128..a4a366773 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -541,7 +541,7 @@ class ExportDistrictRongHe extends BaseBatch{ allRows.addAll(rets2) }else{ def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where CITYID='${cityCode}' and dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";// - allRows = createDeal(sql) + allRows.addAll(createDeal(sql)) } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy index 78537bc39..0722c4464 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy @@ -24,7 +24,7 @@ class FzDataResultStat { static main(args) { def task = new FzDataResultStat() -// task.priceMarkupRun(); + task.priceMarkupRun(); task.regionPriceTrendRun() //// task.margeExcel() // task.saveCsvData() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index 43203f42f..bcbe886de 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202209.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202210.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DataUtil.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DataUtil.groovy index b18f43f40..9d9f4c2d0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DataUtil.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DataUtil.groovy @@ -109,7 +109,8 @@ public class DataUtil { //System.out.println(isCellPhoneNumber("15221169682")); //System.out.println(isCellPhoneNumber("12345611123")); //println getRound(2,125.269); - println formateAddress("苏州","江苏路56号"); +// println formateAddress("苏州","江苏路56号"); + } public static String getHanzi(String s) { @@ -243,7 +244,7 @@ public class DataUtil { return ""; } String regEx = "<[^>]*(>)?"; - return str.replaceAll(regEx, ""); + return str.replaceAll(regEx, " "); } public static Pattern nameTags = Pattern.compile(']*)(\\s|$|>)'); From 8aa49a7b9801e4dbfca345c4f63db7bb7e21af1e Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 11 Nov 2022 11:53:30 +0800 Subject: [PATCH 034/112] 40-80 --- .../script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index 638323257..8b61f2f82 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -25,7 +25,7 @@ class BigdataCitypriceBatch extends BaseBatch{ public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"]; - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) static main(args) { runBatch(new BigdataCitypriceBatch(), args) From 3a9c0f474e814684cb8bafa102187cdb4c53c716 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 11 Nov 2022 14:34:18 +0800 Subject: [PATCH 035/112] =?UTF-8?q?=E6=8D=A2=E6=88=90=E5=B8=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index 8b61f2f82..d0b68ab96 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -33,7 +33,7 @@ class BigdataCitypriceBatch extends BaseBatch{ @Override void process(Object args) { - def citys = CcbCommonUtils.getRunningCitys(args) + def citys = ["杭州","宁波","温州","嘉兴","湖州","绍兴","金华","台州"]//CcbCommonUtils.getRunningCitys(args) def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger(); def endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger(); def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() From b73a97fcab7c146bcb8c19280255bb780cf10720 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 11 Nov 2022 16:35:33 +0800 Subject: [PATCH 036/112] update bug --- .../ronghe/BigdataCitypriceBatch.groovy | 32 +++++++++++++++---- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index d0b68ab96..80e99bd54 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -33,6 +33,7 @@ class BigdataCitypriceBatch extends BaseBatch{ @Override void process(Object args) { + // def citys = ["杭州","宁波","温州","嘉兴","湖州","绍兴","金华","台州"]//CcbCommonUtils.getRunningCitys(args) def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger(); def endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger(); @@ -174,11 +175,11 @@ class BigdataCitypriceBatch extends BaseBatch{ if(!sellMap || (sellMap && sellMap.avg_price.size()<3)){ sellDataType = 3 sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -3), "yyyyMMdd").toInteger(); - sellMap = findHouses(dist.city, dist.region, "sell", startDate, endDate, dist.name, dist.alias, dist.estate_type2) + sellMap = findHouses(dist.city, dist.region, "sell", sellStartDate, endDate, dist.name, dist.alias, dist.estate_type2) if(!sellMap || (sellMap && sellMap.avg_price.size()<3)){ sellDataType = 6 sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -6), "yyyyMMdd").toInteger(); - sellMap = findHouses(dist.city, dist.region, "sell", startDate, endDate, dist.name, dist.alias, dist.estate_type2) + sellMap = findHouses(dist.city, dist.region, "sell", sellStartDate, endDate, dist.name, dist.alias, dist.estate_type2) } } if(sellMap){ @@ -189,6 +190,7 @@ class BigdataCitypriceBatch extends BaseBatch{ sell_avg_total = avgValue(sellMap.total_price) }else{ sell_count = 0 + sellDataType = null } } @@ -201,11 +203,11 @@ class BigdataCitypriceBatch extends BaseBatch{ if(!rentMap || (rentMap && rentMap.avg_price.size()<3)){ rentDataType = 3 rentStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -3), "yyyyMMdd").toInteger(); - rentMap = findHouses(dist.city, dist.region, "rent", startDate, endDate, dist.name, dist.alias, dist.estate_type2) + rentMap = findHouses(dist.city, dist.region, "rent", rentStartDate, endDate, dist.name, dist.alias, dist.estate_type2) if(!rentMap || (rentMap && rentMap.avg_price.size()<3)){ rentDataType = 6 rentStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -6), "yyyyMMdd").toInteger(); - rentMap = findHouses(dist.city, dist.region, "rent", startDate, endDate, dist.name, dist.alias, dist.estate_type2) + rentMap = findHouses(dist.city, dist.region, "rent", rentStartDate, endDate, dist.name, dist.alias, dist.estate_type2) } } if(rentMap){ @@ -216,6 +218,7 @@ class BigdataCitypriceBatch extends BaseBatch{ rent_avg_total = avgValue(rentMap.total_price) }else{ rent_count = 0 + rentDataType = null } } if (sell_avg_price || rent_avg_price) { @@ -263,9 +266,24 @@ class BigdataCitypriceBatch extends BaseBatch{ } else { row << '' } - row << '11' - row << sellDataType - row << rentDataType + if(dist.estate_type2 && dist.estate_type2=="商住"){ + row << '11' + }else{ + row << '13' + } + + if(sellDataType){ + row << sellDataType + }else{ + row << '' + } + + if(rentDataType){ + row << rentDataType + }else{ + row << '' + } + row << 1 row << 'FJW' return row From cffb606a31d6e4f4b5e018cae6bbc0162bb9a463 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 11 Nov 2022 18:19:54 +0800 Subject: [PATCH 037/112] =?UTF-8?q?=E8=B7=AF=E4=B8=8E=E8=B7=AF=E5=8F=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../BigdataCommunityBatch.groovy | 36 ++++++++++--------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index b9a415811..bfce0b81b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -183,23 +183,25 @@ class BigdataCommunityBatch extends BaseBatch{ ret << "" } }else{ - def jsonObj = FangjiaClientUtils.addressStructure(dist.city, dist.address) - if(jsonObj && jsonObj.result && jsonObj.result.street){ - if(jsonObj.result.street){ - ret << jsonObj.result.street - }else{ - ret << "" - } - - if(jsonObj.result.streetNumber){ - ret << jsonObj.result.streetNumber - }else{ - ret << "" - } - }else{ - ret << "" - ret << "" - } + ret << "" + ret << "" +// def jsonObj = FangjiaClientUtils.addressStructure(dist.city, dist.address) +// if(jsonObj && jsonObj.result && jsonObj.result.street){ +// if(jsonObj.result.street){ +// ret << jsonObj.result.street +// }else{ +// ret << "" +// } +// +// if(jsonObj.result.streetNumber){ +// ret << jsonObj.result.streetNumber +// }else{ +// ret << "" +// } +// }else{ +// ret << "" +// ret << "" +// } } }else{ ret << "" From 8ad3456adb95b80b9bccf7df963cd747a39f2d29 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 11 Nov 2022 19:02:00 +0800 Subject: [PATCH 038/112] =?UTF-8?q?=E8=B7=91252=E8=9E=8D=E5=90=88=EF=BC=8C?= =?UTF-8?q?=E5=85=88=E4=BB=8E236=E5=88=87=E5=88=B094?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index d9d7660c0..c126ea1ab 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -42,7 +42,7 @@ class CcbCommonUtils { public static DSCollection dataStoreConnection = null; - public static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + public static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") From 062ce020c7ffa87741391d0ea592ce326dcce5bd Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 11 Nov 2022 19:13:24 +0800 Subject: [PATCH 039/112] =?UTF-8?q?=E8=9E=8D=E5=90=88=E6=A5=BC=E6=A0=8B?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- projects/fangjia-coresite/pom.xml | 8 --- .../ccbevaluation/BigdataBuildingBatch.groovy | 2 +- .../ExportOracleAllDataTest.groovy | 58 +++++++++---------- .../ccbevaluation/GuiDingPriceBatch.groovy | 2 +- .../fangjia/script/ccbevaluation/Test.groovy | 26 ++++++++- 5 files changed, 55 insertions(+), 41 deletions(-) diff --git a/projects/fangjia-coresite/pom.xml b/projects/fangjia-coresite/pom.xml index 72cefc4b9..05fa7f275 100644 --- a/projects/fangjia-coresite/pom.xml +++ b/projects/fangjia-coresite/pom.xml @@ -242,13 +242,5 @@ 1.0.0 jdk18 - - - - - - - - diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy index 3b92e612c..7500406cc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy @@ -23,7 +23,7 @@ class BigdataBuildingBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.citys_all; + def citys = CcbCommonUtils.getRunningCitys(args) def rows = [] def headers = [] for (def city : citys) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy index 145aad4eb..354473ed7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy @@ -42,37 +42,37 @@ class ExportOracleAllDataTest { def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") def new_dists = [] //不会变 10.25 - exportEvaluationRateData() - exportBigdataPoiData()// - exportBigdataDistrictData()// +// exportEvaluationRateData() +// exportBigdataPoiData()// +// exportBigdataDistrictData()// exportGuidedPrice()//随人工改变// - exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策// - - //处理小区情况 - exportBigdataCommunity()// - exportCommunityStaging()//没cityid - exportCommunityBuileType()// - - exportThirdPartyPrice2(new_dists)// - exportBigdataForsale(new_dists) //2022-07-19 - exportBigdataDeal( new_dists) //2022-07-01 - - //第三方网站权重表 - exportWebsiteRate(); - - //计算结果相关 - exportAvmEvaluationGJNB()// - exportAVMEvaluationItemGJNB()// - - exportCityPrice()// - exportRegionPrice()// - - exportCommunityBasePriceGj()// - exportCommunityBasePrice()// - - exportCommunityPdescribeGj()// - exportCommCaseDetailGj()// +// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策// +// +// //处理小区情况 +// exportBigdataCommunity()// +// exportCommunityStaging()//没cityid +// exportCommunityBuileType()// +// +// exportThirdPartyPrice2(new_dists)// +// exportBigdataForsale(new_dists) //2022-07-19 +// exportBigdataDeal( new_dists) //2022-07-01 +// +// //第三方网站权重表 +// exportWebsiteRate(); +// +// //计算结果相关 +// exportAvmEvaluationGJNB()// +// exportAVMEvaluationItemGJNB()// +// +// exportCityPrice()// +// exportRegionPrice()// +// +// exportCommunityBasePriceGj()// +// exportCommunityBasePrice()// +// +// exportCommunityPdescribeGj()// +// exportCommCaseDetailGj()// def curDate = curMonth.toString()+"28" // ok文件 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index bf34601ed..66e8081dd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-09") + def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-11") if(prices){ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 6326f35f4..915f57f71 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -447,8 +447,30 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business ////},0) // -ExcelUtils.modify("D:\\数据\\建行\\二期\\重庆无效小区_3159.xlsx",{d-> -},0) +//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\二期\\11.11\\删除小区(2).xlsx",{d-> +// if(d['A']){ +// def dbo = CommonDatautils.findDistrictById(d['C']) +// if(dbo){ +// def udata = new BasicDBObject() +// udata.put("name_cat", 11) +// udata.put("status", 1) +// udata.put("m_date",new Date()) +// +// def q = new BasicDBObject() +// q.put("_id", dbo._id) +// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +// }else{ +// println dbo._id.toString() +" 没找到" +// } +// } +//},0) + +def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"] +for(def c : citys){ + def districts = CommonDatautils.findAllDistrictByCity(c, "district", ["city","name"]) + println c +" "+districts.size() +} + // // From 83f597e74eaea83192b7e26a7523f4e2336acdb4 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 14 Nov 2022 10:39:15 +0800 Subject: [PATCH 040/112] =?UTF-8?q?=E6=B5=8B=E8=AF=95=E4=BC=B0=E5=80=BC?= =?UTF-8?q?=E4=B8=8E=E8=9E=8D=E5=90=88=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ExportOracleAllDataTest.groovy | 4 +- .../ronghe/ExportDistrictRongHeTest.groovy | 529 ++++++++++-------- 2 files changed, 311 insertions(+), 222 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy index 354473ed7..be746b3c0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy @@ -46,11 +46,11 @@ class ExportOracleAllDataTest { // exportBigdataPoiData()// // exportBigdataDistrictData()// - exportGuidedPrice()//随人工改变// +// exportGuidedPrice()//随人工改变// // exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策// // // //处理小区情况 -// exportBigdataCommunity()// + exportBigdataCommunity()// // exportCommunityStaging()//没cityid // exportCommunityBuileType()// // diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy index 11270dab3..24d6e007e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy @@ -5,51 +5,48 @@ import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher import com.fangjia.script.ccbevaluation.CcbCommonUtils import com.fangjia.script.ccbevaluation.CcbFtpTestUtils - import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.util.DateUtils import groovy.sql.Sql -import java.sql.ResultSet -import java.sql.ResultSetMetaData - @BatchOwner(Watcher.GaoJuan) class ExportDistrictRongHeTest extends BaseBatch{ - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") static main(args) { runBatch(new ExportDistrictRongHeTest(), args); } + def citys = [] + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + @Override void process(Object args) { - def curDate = 20220310//DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") - def fileName = "/ccb_rh/${curDate}" -// if(CommonDatautils.isWindow()){ -// fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}" + citys = CcbCommonUtils.getRunningCitys(args) + def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") + def new_dists = []//["350604","350605","410307","410308","450181"] + //按天导(每天一个文件夹) + exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) + +// if(curDate.toString().endsWith("10")){ + exportBigdataCommunityGJRH()// + exportBigdataBuildingFJW()// + exportBigdataCommfeeGj()// + + exportBigdataCommbuildstruGJ()// + exportBigdataCommhousetypeGJ()// + exportBigdataCommbuildyearGj()// + exportBigdataCommhouseuseGJ()// + exportBigdataPositionGJ()// + exportBigdataDistrictFJW()// + exportBigdataAliasFJW()// + + exportBigdataBuildprojFjw() + exportBigdataCommlayoutpicFjw() + exportEvaluateRate() // } - CcbFtpTestUtils.delete(fileName) - - exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd")) - - exportBigdataCommunityGJRH() - exportBigdataBuildingFJW() - exportBigdataCommfeeGj() - - exportBigdataCommbuildstruGJ() - exportBigdataCommhousetypeGJ() - exportBigdataCommbuildyearGj() - exportBigdataCommhouseuseGJ() - exportBigdataPositionGJ() - exportBigdataDistrictFJW() - exportBigdataAliasFJW() - - exportBigdataBuildprojFjw() - exportBigdataCommlayoutpicFjw() - exportEvaluateRate() - SSHHelper jschCommand = new SSHHelper(); System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); @@ -98,57 +95,75 @@ class ExportDistrictRongHeTest extends BaseBatch{ // exportOk() // } - def exportBigdataCommunityGJRH(cityCode){ - def sql = "select CITYID,DISTID,COMMID,COMMNAME,ADDRESS,LANDAREA,BUILDAREA,GREENINGRATE,PLOTRATIO,PARKINGCNT,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE,COMMALIAS,STREET,STREETNO,IFNEW,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,SOURCECODE from BIGDATA_COMMUNITY_FJW";// where CITYID='${cityCode}' - def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.COMMID - data << r.COMMNAME - data << r.COMMALIAS - data << r.DISTID - data << r.STREET - data << r.STREETNO - data << r.ADDRESS - data << r.LANDAREA - data << r.BUILDAREA - data << r.COMMTYPE - data << r.IFNEW - data << r.GREENINGRATE - data << r.PLOTRATIO - data << r.PARKINGCNT - data << '' - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + def exportBigdataCommunityGJRH(){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" } - FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + def index = 0 + for(def city : citys){ + def allRows = [] + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID,DISTID,COMMID,COMMNAME,ADDRESS,LANDAREA,BUILDAREA,GREENINGRATE,PLOTRATIO,PARKINGCNT,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE,COMMALIAS,STREET,STREETNO,IFNEW,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,SOURCECODE from BIGDATA_COMMUNITY_FJW where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.COMMNAME + data << r.COMMALIAS + data << r.DISTID + data << r.STREET + data << r.STREETNO + data << r.ADDRESS + data << r.LANDAREA + data << r.BUILDAREA + data << r.COMMTYPE + data << r.IFNEW + data << r.GREENINGRATE + data << r.PLOTRATIO + data << r.PARKINGCNT + data << '' + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + if(allRows && allRows.size()>0){ + index = index + 1 + if(index == 1){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + } + } + + } println "BIGDATA_COMMUNITY_FJW 数据文件输出完成......" CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) println "BIGDATA_COMMUNITY_FJW 上传Ftp完成......" } def exportBigdataCommbuildstruGJ(){ - def sql = "select CITYID,COMMID,BUILDSTRU,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDSTRU_FJW";// where CITYID='${cityCode}' def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.COMMID - data << r.BUILDSTRU - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID,COMMID,BUILDSTRU,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDSTRU_FJW where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.BUILDSTRU + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ @@ -159,20 +174,25 @@ class ExportDistrictRongHeTest extends BaseBatch{ println "BIGDATA_COMMBUILDSTRU_FJW 上传Ftp完成......" } - def exportBigdataCommhousetypeGJ(cityCode){ - def sql = "select CITYID,COMMID,HOUSETYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSETYPE_FJW";// where CITYID='${cityCode}' + def exportBigdataCommhousetypeGJ(){ + def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.COMMID - data << r.HOUSETYPE - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID,COMMID,HOUSETYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSETYPE_FJW where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.HOUSETYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ @@ -184,19 +204,23 @@ class ExportDistrictRongHeTest extends BaseBatch{ } def exportBigdataCommbuildyearGj(){ - def sql = "select CITYID,COMMID,BUILDYEAR,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDYEAR_FJW";// where CITYID='${cityCode}' def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.COMMID - data << r.BUILDYEAR - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID,COMMID,BUILDYEAR,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDYEAR_FJW where CITYID='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.BUILDYEAR + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ @@ -208,29 +232,33 @@ class ExportDistrictRongHeTest extends BaseBatch{ } def exportBigdataCommfeeGj(){ - def sql = "select CITYID,COMMID,COMPID,FEE,HOUSEUSE,BUILDTYPE,FEETYPE,NOTE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMFEE_FJW";//where CITYID='${cityCode}' + def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.COMMID - data << r.COMPID - data << r.FEE - data << r.HOUSEUSE - if(r.BUILDTYPE){ - def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") - data << arrs[0].trim() - }else{ - data << '' - } - data << r.FEETYPE - data << r.NOTE - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys){ + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID,COMMID,COMPID,FEE,HOUSEUSE,BUILDTYPE,FEETYPE,NOTE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMFEE_FJW where CITYID='${cityCode}'"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.COMPID + data << r.FEE + data << r.HOUSEUSE + if(r.BUILDTYPE){ + def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.FEETYPE + data << r.NOTE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ @@ -243,20 +271,24 @@ class ExportDistrictRongHeTest extends BaseBatch{ def exportBigdataCommhouseuseGJ(){ - def sql = "select CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSEUSE_FJW";// where CITYID='${cityCode}' def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.COMMID - data << r.HOUSEUSE - data << r.HOUSECNT - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSEUSE_FJW where CITYID='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.HOUSEUSE + data << r.HOUSECNT + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ @@ -268,24 +300,28 @@ class ExportDistrictRongHeTest extends BaseBatch{ } def exportBigdataPositionGJ(){ - def sql = "select CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_POSITION_FJW";// where CITYID='${cityCode}' def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.OBJECTTYPE - data << r.OBJECTID - data << r.POSITIONTYPE - data << r.SEQNO - data << r.LONGITUDE - data << r.LATITUDE - data << r.ALTITUDE - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_POSITION_FJW where CITYID='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.OBJECTTYPE + data << r.OBJECTID + data << r.POSITIONTYPE + data << r.SEQNO + data << r.LONGITUDE + data << r.LATITUDE + data << r.ALTITUDE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ @@ -297,23 +333,27 @@ class ExportDistrictRongHeTest extends BaseBatch{ } def exportBigdataDistrictFJW(){ - def sql = "select DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_DISTRICT_FJW";// where CITYID='${cityCode}' def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.DISTID - data << r.DISTNAME - data << r.DISTSPELL - data << r.CITYID - data << r.DISTCODE - data << r.SEQ - data << r.DISTTYPE - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_DISTRICT_FJW where CITYID='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.DISTID + data << r.DISTNAME + data << r.DISTSPELL + data << r.CITYID + data << r.DISTCODE + data << r.SEQ + data << r.DISTTYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ @@ -325,26 +365,30 @@ class ExportDistrictRongHeTest extends BaseBatch{ } def exportBigdataAliasFJW(){ - def sql = "select city_id, ca_id,address_id,address_name, status, exp_dt, version, is_key_name, data_source, description,alias_grade,addr_type_cd,ifexamine,ifdisplay from ADDRESS_ALIAS_FJW";// where city_id='${cityCode}' def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.city_id - data << r.ca_id - data << r.address_id - data << r.address_name - data << r.status - data << r.exp_dt - data << r.version - data << r.is_key_name - data << r.data_source - data << r.description - data << r.alias_grade - data << r.addr_type_cd - data << r.ifexamine - data << r.ifdisplay - allRows << data - }); + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select city_id, ca_id,address_id,address_name, status, exp_dt, version, is_key_name, data_source, description,alias_grade,addr_type_cd,ifexamine,ifdisplay from ADDRESS_ALIAS_FJW where city_id='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.city_id + data << r.ca_id + data << r.address_id + data << r.address_name + data << r.status + data << r.exp_dt + data << r.version + data << r.is_key_name + data << r.data_source + data << r.description + data << r.alias_grade + data << r.addr_type_cd + data << r.ifexamine + data << r.ifdisplay + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ @@ -464,7 +508,7 @@ class ExportDistrictRongHeTest extends BaseBatch{ } - def exportBigdataDealFJW2(curDate, dealdate){ + def exportBigdataDealFJW2(curDate, dealdate, new_dists){ def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd") def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd") @@ -476,7 +520,39 @@ class ExportDistrictRongHeTest extends BaseBatch{ } def allRows = [] - def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and + +// for(def city : citys) { +// def cityCode = cityCodeMap.get(city) +// if(curDate.toString().endsWith("28")){ +// if(new_dists){ +// def newstr = "('"+new_dists.join("','")+"')" +// def sql1 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ " + +// "where CITYID='${cityCode}' and dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd') and" + +// " distid not in ${newstr} ";//CITYID='${cityCode}' and +// def rets1 = createDeal(sql1) +// +// def sql2 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ " + +// " where CITYID='${cityCode}' and distid in ${newstr}" +// def rets2 = createDeal(sql2) +// allRows.addAll(rets1) +// allRows.addAll(rets2) +// }else{ +// def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where CITYID='${cityCode}' and dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";// +// allRows.addAll(createDeal(sql)) +// } +// } +// } + + def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ ";// + allRows.addAll(createDeal(sql)) + + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_DEAL_FJW 上传Ftp完成......${endDateStr}" + } + + def createDeal(sql){ + def allRows = [] sqlInstance.eachRow(sql,{r-> def data = [] data << r.CITYID @@ -550,9 +626,8 @@ class ExportDistrictRongHeTest extends BaseBatch{ data << r.WORKDATE allRows << data }); - FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); - CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) - println "BIGDATA_DEAL_FJW 上传Ftp完成......${endDateStr}" + + return allRows; } def exportAllBigdataDealFJW(){ @@ -641,45 +716,59 @@ class ExportDistrictRongHeTest extends BaseBatch{ } - def exportBigdataBuildingFJW(cityCode){ - def sql = "select CITYID, BUILDID, BUILDNO, BUILDNAME, BUILDALIAS, DISTID, COMMID, BUILDYEAR, BUILDTYPE, BUILDSTRU, FLOORCNT, UNITCNT, HOUSECNT, IFELEVATOR, ELEVATORCNT, ELEVATORHOUSE, REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, VALID_FLAG from BIGDATA_BUILDING_FJW";//where CITYID='${cityCode}' - def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.BUILDID - data << r.BUILDNO - data << r.BUILDNAME - data << r.BUILDALIAS - data << r.DISTID - data << r.COMMID - data << r.BUILDYEAR - if(r.BUILDTYPE){ - def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") - data << arrs[0].trim() - }else{ - data << '' - } - data << r.BUILDSTRU - data << r.FLOORCNT - data << r.UNITCNT - data << r.HOUSECNT - data << r.IFELEVATOR - data << r.ELEVATORCNT - data << r.ELEVATORHOUSE - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - data << r.VALID_FLAG - allRows << data - }); + def exportBigdataBuildingFJW(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" } - FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + def index = 0 + for(def city : citys) { + + def allRows = [] + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID, BUILDID, BUILDNO, BUILDNAME, BUILDALIAS, DISTID, COMMID, BUILDYEAR, BUILDTYPE, BUILDSTRU, FLOORCNT, UNITCNT, HOUSECNT, IFELEVATOR, ELEVATORCNT, ELEVATORHOUSE, REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, VALID_FLAG from BIGDATA_BUILDING_FJW where CITYID='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.BUILDALIAS + data << r.DISTID + data << r.COMMID + data << r.BUILDYEAR + if(r.BUILDTYPE){ + def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.BUILDSTRU + data << r.FLOORCNT + data << r.UNITCNT + data << r.HOUSECNT + data << r.IFELEVATOR + data << r.ELEVATORCNT + data << r.ELEVATORHOUSE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.VALID_FLAG + allRows << data + }); + + if(allRows && allRows.size()>0){ + index = index + 1 + if(index == 1){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + } + } + } + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) println "BIGDATA_BUILDING_FJW 上传Ftp完成......" } From 1e055ad8969d7853e09cc22a83041bd6ff1fe102 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 14 Nov 2022 15:46:53 +0800 Subject: [PATCH 041/112] =?UTF-8?q?=E5=8E=BB=E9=99=A4=E7=89=A9=E4=B8=9A?= =?UTF-8?q?=E7=B1=BB=E5=9E=8B=E6=A3=80=E7=B4=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/thirdPriceCrawl/SynchDistrictPrice.groovy | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index a84057ac1..57c47a013 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -25,7 +25,7 @@ class SynchDistrictPrice extends BaseBatch{ def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"] // "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产" - def esSources = [ "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产"] + def esSources = [ "搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产"] static main(args) { @@ -145,7 +145,7 @@ class SynchDistrictPrice extends BaseBatch{ println "["+source +"-"+city + "] add:"+ rets.size()+" 原抓取:"+count }else{ for(def r : rets){ - def obj= findIsExists(r.city, r.region, r.name, r.estate_type, SHORT_DATE, source) + def obj= findIsExists(r.city, r.region, r.name, SHORT_DATE, source) if(obj){ updateIndex++ // thirdPartyColl.remove(new BasicDBObject('_id',obj._id)) @@ -183,16 +183,14 @@ class SynchDistrictPrice extends BaseBatch{ } - def findIsExists(def city, def region, def name, estate_type, def sDate, def source){ + def findIsExists(def city, def region, def name, def sDate, def source){ def q = new BasicDBObject() q.put("source", source) q.put("s_date", sDate) q.put("city", city) q.put("region", region) q.put("name", name) - if(estate_type){ - q.put("estate_type",estate_type) - } + def obj = thirdPartyColl.findOne(q) return obj } From d9632d2c679e7f5cf486b2810bba1055919570bc Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 14 Nov 2022 18:34:32 +0800 Subject: [PATCH 042/112] update --- .../script/ccbevaluation/BigdataDealBatch.groovy | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index a1a81702d..00771ac49 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -88,13 +88,16 @@ class BigdataDealBatch extends BaseBatch{ def trades = getCityData(city, statMap, ppeDists) - def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") - rows.addAll(rets) - println "【${index}】"+city +" 成交共:"+rets.size() + if(trades){ + def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") + rows.addAll(rets) + println "【${index}】"+city +" 成交共:"+rets.size() + + def cityDatas = [] + cityDatas << headers + cityDatas.addAll(rows) + } - def cityDatas = [] - cityDatas << headers - cityDatas.addAll(rows) } def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" From 7b879c11df54e70c3df8aa6a521d711d1d0f93e0 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 15 Nov 2022 09:58:17 +0800 Subject: [PATCH 043/112] =?UTF-8?q?=E6=95=B0=E6=8D=AE=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/ExportOracleAllDataTest.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy index be746b3c0..22ffc122b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy @@ -50,13 +50,13 @@ class ExportOracleAllDataTest { // exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策// // // //处理小区情况 - exportBigdataCommunity()// +// exportBigdataCommunity()// // exportCommunityStaging()//没cityid -// exportCommunityBuileType()// -// -// exportThirdPartyPrice2(new_dists)// -// exportBigdataForsale(new_dists) //2022-07-19 -// exportBigdataDeal( new_dists) //2022-07-01 + exportCommunityBuileType()// + + exportThirdPartyPrice2(new_dists)// + exportBigdataForsale(new_dists) //2022-07-19 + exportBigdataDeal( new_dists) //2022-07-01 // // //第三方网站权重表 // exportWebsiteRate(); From 383395581e01aa10e5b4ce323f47bed66e6ffee8 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 15 Nov 2022 15:21:50 +0800 Subject: [PATCH 044/112] =?UTF-8?q?=E8=A1=8C=E6=83=85=E5=B0=8F=E5=8C=BA?= =?UTF-8?q?=E4=BB=B7=E6=A0=BC=E5=A1=AB=E8=A1=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ronghe/BigdataCitypriceBatch.groovy | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index 80e99bd54..ad1d1b2b3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -25,7 +25,7 @@ class BigdataCitypriceBatch extends BaseBatch{ public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"]; - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) static main(args) { runBatch(new BigdataCitypriceBatch(), args) @@ -33,7 +33,6 @@ class BigdataCitypriceBatch extends BaseBatch{ @Override void process(Object args) { - // def citys = ["杭州","宁波","温州","嘉兴","湖州","绍兴","金华","台州"]//CcbCommonUtils.getRunningCitys(args) def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger(); def endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger(); @@ -114,12 +113,21 @@ class BigdataCitypriceBatch extends BaseBatch{ def districtRun(citys, startDate, endDate, cityCodeDataMap){ def rets = [] def allDistricts = [] + def allPriceMap = [:] + def allRentMap = [:] for(def city : citys) { def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias"]); allDistricts.addAll(districts) + + def priceMap = CommonDatautils.queryDistrictStatByDate(city, startDate, endDate) + allPriceMap = allPriceMap + priceMap + + def rentMap = CommonDatautils.findSqmetreRentByCityAndDate(city, endDate) + allRentMap = allRentMap + rentMap + } - def cityLists = districtRunningByCity(allDistricts, startDate, endDate, cityCodeDataMap) + def cityLists = districtRunningByCity(allDistricts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap) if(cityLists){ rets.addAll(cityLists) } @@ -133,7 +141,7 @@ class BigdataCitypriceBatch extends BaseBatch{ println "BIGDATA_COMMPRICE 数据文件输出完成......" } - def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap){ + def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap){ def totalCount = new AtomicLong() def distCount = districts.size() final CountDownLatch latch = new CountDownLatch(distCount) @@ -145,7 +153,7 @@ class BigdataCitypriceBatch extends BaseBatch{ void run() { try { totalCount.addAndGet(1) - def row = createDistDate(dist, startDate, endDate, cityCodeDataMap) + def row = createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap) if(row){ rets << row println totalCount.get() +"/"+distCount + dist.city+" "+dist.region+" "+dist.name+" "+row[4]+" "+row[6] @@ -163,7 +171,7 @@ class BigdataCitypriceBatch extends BaseBatch{ } - def createDistDate(dist, startDate, endDate, cityCodeDataMap){ + def createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap){ def sellDataType = 1 def rentDataType = 1 def sellMap = findHouses(dist.city, dist.region, "sell", startDate, endDate, dist.name, dist.alias, dist.estate_type2) @@ -221,6 +229,12 @@ class BigdataCitypriceBatch extends BaseBatch{ rentDataType = null } } + if(!sell_avg_price){ + sell_avg_price = allPriceMap.get(dist.city +"##"+ dist.region +"##"+ dist.name) + } + if(!rent_avg_price){ + rent_avg_price = allRentMap.get(dist.city +"##"+ dist.region +"##"+ dist.name) + } if (sell_avg_price || rent_avg_price) { def row = [] row << cityCodeDataMap.get(dist.city) From ee3412548e16c20b608cca72466fed9e7b9e70a1 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 15 Nov 2022 15:28:59 +0800 Subject: [PATCH 045/112] update --- .../script/commData/CommonDatautils.groovy | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 3be6ffa9a..8f53bcb0a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1556,6 +1556,32 @@ class CommonDatautils { } return rets; } + + + public static def queryDistrictStatByDate(city, startDate, endDate){ + DBObject q = new BasicDBObject() + q.put("city", city); + q.put('weekend', new BasicDBObject('$gte', DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd")).append('$lte', DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd"))); + def dbcursor = DIST_STAT_COLL.find(q).batchSize(5000) + dbcursor.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def districtStatMap = [:] + for(def d : dbcursor){ + def key = d.city +"##"+d.region+"##"+d.district_name + def prices = districtStatMap.get(key) + if(!prices){ + prices = [] + districtStatMap.put(key, prices) + } + prices << d.avg_price + } + + def priceMap = [:] + districtStatMap.each{k,v-> + def price = v.sum()/v.size() + priceMap.put(k, (int)price) + } + return priceMap; + } public static def queryDistrictPrice(city, region, name, weekend, cat){ @@ -3290,6 +3316,20 @@ class CommonDatautils { return count; } + static def findSqmetreRentByCityAndDate(def city, def weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("cat","district") + q.put("weekend", DateUtils.parseFromTightByPattern(weekend.toString(), "yyyyMMdd")) + def dbcursor = SQMETRE_RENT_COLL.find(q).batchSize(2000) + dbcursor.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def sqmetreRentMap = [:] + for(def d : dbcursor){ + sqmetreRentMap.put(d.city+"##"+d.region+"##"+d.district_name, d.avg_price) + } + return sqmetreRentMap; + } + static def findNewSqmetreRent(def city, def region, def name){ def q = new BasicDBObject() q.put("city", city) From b47da86481ddb6321184487a54cc89590fd276da Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 15 Nov 2022 15:33:14 +0800 Subject: [PATCH 046/112] =?UTF-8?q?=E7=BA=BF=E7=A8=8B=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index ad1d1b2b3..7e7afa70c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -25,7 +25,7 @@ class BigdataCitypriceBatch extends BaseBatch{ public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"]; - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) static main(args) { runBatch(new BigdataCitypriceBatch(), args) From 40e0ebe0c5828ab1f4b073684542c54cadf884ef Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 16 Nov 2022 13:27:00 +0800 Subject: [PATCH 047/112] =?UTF-8?q?=E8=A1=8C=E6=83=85=E5=85=A8=E9=83=A8?= =?UTF-8?q?=E6=94=BE=E5=BC=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/ronghe/BigdataCitypriceBatch.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index 7e7afa70c..90a46050c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -33,13 +33,13 @@ class BigdataCitypriceBatch extends BaseBatch{ @Override void process(Object args) { - def citys = ["杭州","宁波","温州","嘉兴","湖州","绍兴","金华","台州"]//CcbCommonUtils.getRunningCitys(args) + def citys = CcbCommonUtils.getRunningCitys(args) def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger(); def endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger(); def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() -// cityRun(citys, startDate, endDate, cityCodeDataMap) -// regionRun(citys, startDate, endDate, cityCodeDataMap) + cityRun(citys, startDate, endDate, cityCodeDataMap) + regionRun(citys, startDate, endDate, cityCodeDataMap) districtRun(citys, startDate, endDate, cityCodeDataMap) } From d619f0928703c29ff98650b0efd00fed3ed63bd6 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 16 Nov 2022 17:14:47 +0800 Subject: [PATCH 048/112] =?UTF-8?q?=E8=A1=A5=E5=85=8515=E4=B8=AA=E5=9F=8E?= =?UTF-8?q?=E5=B8=82=E7=9A=84=E7=AC=AC=E4=B8=89=E6=96=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/AddThirdPartyPrice.groovy | 88 +++++++ .../ExportOracleAllDataTest.groovy | 6 +- .../fangjia/script/ccbevaluation/Test.groovy | 67 ++++-- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 18 +- .../ronghe/ExportDistrictRongHeTest.groovy | 217 +++++++++++++----- .../script/commData/CommonDatautils.groovy | 16 ++ .../fangjia/script/gj/YouDaDataParser.groovy | 6 +- .../gj/paimai/BeiKeCrawlByChrome.groovy | 4 +- .../gj/paimai/LianjiaCrawlByChrome.groovy | 2 +- .../script/thirdPriceCrawl/DataStat.groovy | 8 +- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 5 +- 11 files changed, 337 insertions(+), 100 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPartyPrice.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPartyPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPartyPrice.groovy new file mode 100644 index 000000000..7a641d018 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPartyPrice.groovy @@ -0,0 +1,88 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import groovy.sql.Sql +import org.bson.types.ObjectId + +class AddThirdPartyPrice { + + public static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + static main(args) { + def task = new AddThirdPartyPrice() + task.run() + } + + def run(){ +//"海西州","天门","大兴安岭","儋州",,"鹤岗","日喀则","吐鲁番", "阿勒泰","迪庆州","潜江","塔城","克孜勒苏","和田","嘉峪关" + def citys = ["林芝"] + def rows = [] + def cityCodeDatas = CcbCommonUtils.findCityCodeValue() + for(def c : citys){ + println c +" "+cityCodeDatas.get(c) + def dists = CcbCommonUtils.findAllDistrictByCity(c, "district", ["city", "region", "name","name_cat", "estate_type2","lng"]) + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(c) + def rets = createDate(dists, distCodeMap) + println dists.size() +" "+rets.size() + if(rets){ + rows.addAll(rets) + } + } + + def fileName = "/home/goojia/ccb/BIGDATA_THIRD_PARTY_PRICE_GJ.dat" + if(CommonDatautils.isWindow()){ + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_THIRD_PARTY_PRICE_GJ.dat" + } + + def header = [] + FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false); + } + + def createDate(dists, distCodeMap){ + def cityCodeDatas = CcbCommonUtils.findCityCodeValue() + + def results = [] + for(def d : dists){ + + def cityId = cityCodeDatas.get(d.city) + def data = distCodeMap.get(d.region) + def avgPrice = findGuidedPrice(cityId, d._id.toString()) + if(!avgPrice){ + def dbo = CommonDatautils.queryHouseCalutePrice(d.city, d.region, d.name, DateUtils.parseFromTightYYYYMMDD("2022-05-01")) + if(dbo){ + avgPrice = dbo.avg_price + } + } + if(avgPrice){ + def ret = [] + ret << new ObjectId().toString() + ret << cityId + ret << data.district_code + ret << d._id.toString() + ret << d.name + ret << "18" + ret << "21世纪" + ret << 202209 + ret << avgPrice + ret << "" + ret << "" + ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd") + results << ret + } + } + return results; + } + + + def findGuidedPrice(cityId, commid){ + def sql = "select avg_price from guided_price_gj where cityid='${cityId}' and commid='${commid}' and dealdate>=to_date('2022-01-01','yyyy-MM-dd')" + def avg_price = null + sqlInstance.eachRow(sql,{r-> + avg_price = r.avg_price + }); + return avg_price + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy index 22ffc122b..71c378532 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy @@ -52,11 +52,11 @@ class ExportOracleAllDataTest { // //处理小区情况 // exportBigdataCommunity()// // exportCommunityStaging()//没cityid - exportCommunityBuileType()// +// exportCommunityBuileType()// exportThirdPartyPrice2(new_dists)// - exportBigdataForsale(new_dists) //2022-07-19 - exportBigdataDeal( new_dists) //2022-07-01 +// exportBigdataForsale(new_dists) //2022-07-19 +// exportBigdataDeal( new_dists) //2022-07-01 // // //第三方网站权重表 // exportWebsiteRate(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 915f57f71..f8e04a350 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -33,8 +33,8 @@ import java.util.regex.*; def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") -//def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", -// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") +def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") @@ -465,11 +465,35 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business // } //},0) -def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"] -for(def c : citys){ - def districts = CommonDatautils.findAllDistrictByCity(c, "district", ["city","name"]) - println c +" "+districts.size() -} +//def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"] +//for(def c : citys){ +// def districts = CommonDatautils.findAllDistrictByCity(c, "district", ["city","name"]) +// println c +" "+districts.size() +//} + +def cityCodeMap = CcbCommonUtils.findCityCodeValue() +ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\252城第三方均价.xlsx", { d -> +// cityCodeMap.each {k,v-> +// if(d['A'].toString()==v){ +// d['C'] = k +// println k +// } +// } +// if(!d['D']){ +// d['D'] = findCount(d['A'].toString().toDouble().toInteger().toString(), sqlInstance94) +// } + d['F'] = findThirdCommCount(d['A'].toString().toDouble().toInteger().toString(), sqlInstance94) + +}); + +//def citys = CcbCommonUtils.citys_252 +//def runCitys = ['聊城','宝鸡','韶关','阜阳','宜春','周口','河源','忻州','三门峡','永州','衢州','盘锦','白银','玉溪','达州','景德镇','鹤壁','辽阳','六盘水','张家界','海西州','兴安盟','阿坝州','那曲','西双版纳州','辽源','邯郸','平顶山','南阳','商丘','南平','黄冈','鄂尔多斯','泸州','六安','葫芦岛','潮州','宿州','庆阳','怀化','张掖','黔东南','云浮','本溪','百色','伊春','昭通','锡林郭勒盟','克拉玛依','天门','博尔塔拉','大兴安岭','潍坊','绵阳','梅州','襄阳','包头','枣庄','东营','信阳','晋中','开封','承德','茂名','荆门','红河','陇南','铁岭','黄山','咸宁','延安','广元','乌兰察布','随州','益阳','阳泉','内江','酒泉','毕节','安顺','丽江','商洛','来宾','黔西南','防城港','铜仁','儋州','崇左','衡阳','咸阳','新乡','南充','荆州','德阳','汉中','湘潭','赤峰','佳木斯','丹东','淮南','抚州','丽水','广安','亳州','昌吉','鄂州','白城','恩施','伊犁','铜陵','林芝','海东','桂林','临汾','渭南','滨州','菏泽','孝感','营口','濮阳','抚顺','自贡','新余','鸡西','钦州','淮北','安康','资阳','乌海','中卫','鹤岗','石嘴山','日喀则','昌都','德宏州','吐鲁番','阿勒泰','临沂','威海','泰安','宿迁','德州','安阳','驻马店','绥化','上饶','莆田','蚌埠','大庆','漯河','十堰','晋城','宣城','舟山','邵阳','大理','楚雄','天水','池州','松原','阜新','巴音郭楞','仙桃','武威','普洱','河池','铜川','迪庆州','潜江','贺州','塔城','济宁','江门','赣州','许昌','滁州','龙岩','运城','长治','乐山','郴州','遂宁','曲靖','呼伦贝尔','阳江','朝阳','巴彦淖尔','朔州','娄底','梧州','巴中','黔南','吴忠','临夏','凉山州','定西','萍乡','甘南','攀枝花','克孜勒苏','山南','临沧','金昌','邢台','衡水','宁德','榆林','鞍山','焦作','牡丹江','齐齐哈尔','眉山','三明','吕梁','吉安','平凉','通辽','揭阳','黄石','宜宾','延边','汕尾','玉林','通化','四平','阿克苏','贵港','白山','双鸭山','湘西','哈密','保山','鹰潭','喀什','雅安','黑河','和田','固原','阿拉善盟','七台河','嘉峪关'] +//for(def c : citys){ +// if(!runCitys.contains(c)){ +// def code = cityCodeMap.get(c) +// println code +" "+ c +// } +//} // @@ -757,15 +781,26 @@ for(def c : citys){ // return count //} // -//def findCount(cityid, sqlInstance){ -// def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0"; -// def count = 0 -// -// sqlInstance.eachRow(sql,{r-> -// count = r.count -// }); -// return count -//} +def findCount(cityid, sqlInstance){ + def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0"; + def count = 0 + + sqlInstance.eachRow(sql,{r-> + count = r.count + }); + return count +} + + +def findThirdCommCount(cityid, sqlInstance){ + def sql = "select count(*) as count from (select count(*) from bigdata_third_party_price_gj where price_date=202209 and cityid='${cityid}' group by commid)"; + def count = 0 + + sqlInstance.eachRow(sql,{r-> + count = r.count + }); + return count +} // // // diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 9fd693e39..7637891ea 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202209//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202207 - def endDate = 202209 + def sDate= 202204//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202202 + def endDate = 202204 // def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") @@ -46,10 +46,10 @@ class ThirdPartyPriceBatch extends BaseBatch { def rows = [] def index = 0 def weekend = DateUtils.getFirstDayOfWeek(new Date()) + def cityCodeDatas = CcbCommonUtils.findCityCodeValue() for(def city : citys){ index++ def statMap = getDistrictStatByCity(city, weekend) - def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"]) @@ -502,17 +502,7 @@ class ThirdPartyPriceBatch extends BaseBatch { def findThirdPartyPrice(def city, def statMap){ def q = new BasicDBObject() q.put("city", city) -<<<<<<< .mine - - //q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) - q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) -======= - q.put("s_date", endDate) - // q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) - - ->>>>>>> .theirs def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1)) def map = [:] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy index 24d6e007e..26520d0d5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy @@ -13,7 +13,7 @@ import groovy.sql.Sql @BatchOwner(Watcher.GaoJuan) class ExportDistrictRongHeTest extends BaseBatch{ - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") static main(args) { runBatch(new ExportDistrictRongHeTest(), args); @@ -28,24 +28,31 @@ class ExportDistrictRongHeTest extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //按天导(每天一个文件夹) - exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) + // exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) // if(curDate.toString().endsWith("10")){ - exportBigdataCommunityGJRH()// - exportBigdataBuildingFJW()// - exportBigdataCommfeeGj()// - exportBigdataCommbuildstruGJ()// - exportBigdataCommhousetypeGJ()// - exportBigdataCommbuildyearGj()// - exportBigdataCommhouseuseGJ()// - exportBigdataPositionGJ()// - exportBigdataDistrictFJW()// - exportBigdataAliasFJW()// +// exportBigdataBuildprojFjw() +// exportBigdataCommlayoutpicFjw() +// exportEvaluateRate() + +// exportBigdataCommunityGJRH()// +// exportBigdataBuildingFJW()// +// exportBigdataCommfeeGj()// +// +// exportBigdataCommbuildstruGJ()// +// exportBigdataCommhousetypeGJ()// +// exportBigdataCommbuildyearGj()// +// exportBigdataCommhouseuseGJ()// +// exportBigdataPositionGJ()// +// exportBigdataDistrictFJW()// +// exportBigdataAliasFJW()// + + def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() + exportBigdataCityPrice(queryDate) + exportBigdataDistPrice(queryDate) + exportBigdataCommPrice(queryDate) - exportBigdataBuildprojFjw() - exportBigdataCommlayoutpicFjw() - exportEvaluateRate() // } SSHHelper jschCommand = new SSHHelper(); System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); @@ -54,46 +61,7 @@ class ExportDistrictRongHeTest extends BaseBatch{ } -// -// def runByCity(){ -// def citys = CcbCommonUtils.citys; -// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() -// -// -//// //检查ok文件是否上传 -//// exportAllBigdataDealFJW() -//// for(def city : citys) { -//// def cityCode = cityCodeDataMap.get(city) -//// -// //按天导(一个文件夹下) -// //exportBigdataDealFJW(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd")) -// //按天导(每天一个文件夹) -// exportBigdataDealFJW2(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd")) -// -//// exportBigdataCommunityGJRH(cityCode) -//// exportBigdataBuildingFJW(cityCode) -//// exportBigdataCommfeeGj(cityCode) -//// -//// exportBigdataCommbuildstruGJ(cityCode) -//// exportBigdataCommhousetypeGJ(cityCode) -//// exportBigdataCommbuildyearGj(cityCode) -//// exportBigdataCommhouseuseGJ(cityCode) -//// exportBigdataPositionGJ(cityCode) -//// exportBigdataDistrictFJW(cityCode) -//// exportBigdataAliasFJW(cityCode) -//// -//// exportBigdataBuildprojFjw(cityCode) -//// exportBigdataCommlayoutpicFjw(cityCode) -//// } -//// exportEvaluateRate() -// -//// -// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") -// SSHHelper jschCommand = new SSHHelper(); -// System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); -// -// exportOk() -// } + def exportBigdataCommunityGJRH(){ @@ -773,6 +741,145 @@ class ExportDistrictRongHeTest extends BaseBatch{ println "BIGDATA_BUILDING_FJW 上传Ftp完成......" } + def exportBigdataCityPrice(queryDate){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_CITYPRICE_ADD_${curDate}_0001.dat" + } + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.YMID + data << r.SALEPRICE?r.SALEPRICE:'' + data << r.RENTPRICE?r.RENTPRICE:'' + if(r.SALECOUNT>0) { + data << r.SALECOUNT + }else{ + data << '' + } + if(r.RENTCOUNT>0){ + data << r.RENTCOUNT + }else{ + data << '' + } + data << r.UPDATETIME + data << r.HOUSEUSE + data << r.SALEBLDGAREA?r.SALEBLDGAREA:'' + data << r.RENTBLDGAREA?r.RENTBLDGAREA:'' + data << r.SALETOTALPRICE?r.SALETOTALPRICE:'' + data << r.RENTTOTALPRICE?r.RENTTOTALPRICE:'' + data << r.DATATYPE + data << r.SDATERANGE + data << r.RDATERANGE + data << r.SOURCECODE + allRows << data + }); + } + + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_CITYPRICE 上传Ftp完成......" + } + + + def exportBigdataDistPrice(queryDate){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTPRICE_ADD_${curDate}_0001.dat" + } + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID,DISTID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_DISTPRICE where CITYID='${cityCode}' and YMID=${queryDate} ";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.YMID + data << r.DISTID + data << r.SALEPRICE?r.SALEPRICE:'' + data << r.RENTPRICE?r.RENTPRICE:'' + if(r.SALECOUNT>0) { + data << r.SALECOUNT + }else{ + data << '' + } + if(r.RENTCOUNT>0){ + data << r.RENTCOUNT + }else{ + data << '' + } + data << r.UPDATETIME + data << r.HOUSEUSE + data << r.SALEBLDGAREA?r.SALEBLDGAREA:'' + data << r.RENTBLDGAREA?r.RENTBLDGAREA:'' + data << r.SALETOTALPRICE?r.SALETOTALPRICE:'' + data << r.RENTTOTALPRICE?r.RENTTOTALPRICE:'' + data << r.DATATYPE + data << r.SDATERANGE + data << r.RDATERANGE + data << r.SOURCECODE + allRows << data + }); + } + + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_DISTPRICE 上传Ftp完成......" + } + + + def exportBigdataCommPrice(queryDate){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMPRICE_ADD_${curDate}_0001.dat" + } + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE where CITYID='${cityCode}' and YMID=${queryDate}";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.YMID + data << r.COMMID + data << r.SALEPRICE?r.SALEPRICE:'' + if(r.SALECOUNT>0) { + data << r.SALECOUNT + }else{ + data << '' + } + data << r.RENTPRICE?r.RENTPRICE:'' + if(r.RENTCOUNT>0){ + data << r.RENTCOUNT + }else{ + data << '' + } + data << r.UPDATETIME + data << r.SALEBLDGAREA?r.SALEBLDGAREA:'' + data << r.RENTBLDGAREA?r.RENTBLDGAREA:'' + data << r.SALETOTALPRICE?r.SALETOTALPRICE:'' + data << r.RENTTOTALPRICE?r.RENTTOTALPRICE:'' + data << r.HOUSEUSE + data << r.SDATERANGE + data << r.RDATERANGE + data << r.DATATYPE + data << r.SOURCECODE + allRows << data + }); + } + + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_COMMPRICE 上传Ftp完成......" + } + def exportEvaluateRate(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 8f53bcb0a..49bacf4ee 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1672,6 +1672,22 @@ class CommonDatautils { } + public static def queryHouseCalutePrice(city, region, name, startWeekendDate){ + DBObject queryCondition = new BasicDBObject() + queryCondition.put("city", city); + queryCondition.put("region", region); + queryCondition.put("district_name", name); + queryCondition.put('weekend', new BasicDBObject('$gte', startWeekendDate)); + queryCondition.put("value_type", new BasicDBObject('$in', [1,2])); + DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", 1)).limit(1) + def dbo = null; + while(cur.hasNext()) { + dbo = cur.next() + } + return dbo; + } + + public static def queryDistrictPriceByWeekend(city, region, name, weekend, limit){ DBObject queryCondition = new BasicDBObject() queryCondition.put("city", city); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index 6ccaabde3..6d692218b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -48,13 +48,13 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2022-09-01" + static def statDate = "2022-10-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo -// task.esfParser("D:\\数据\\友达\\2022年10月\\查询1.DBF");//友达二手房存入mongo +// task.esfParser("D:\\数据\\友达\\2022年11月\\查询1.DBF");//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es -// task.esfhouseToSaveEs();//二手房保存到es + task.esfhouseToSaveEs();//二手房保存到es // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index ca99d1f97..f033823f9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -398,13 +398,13 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = e.attr("href") - if(e.text().contains("福州")){ +// if(e.text().contains("福州")){ def cityMap = [:] cityMap.city = e.text() cityMap.url = "https:"+href list << cityMap // cityMap.put(e.text(), "https:"+href) - } +// } } } return list diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 30772c368..0ca1bc62a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -474,7 +474,7 @@ public class LianjiaCrawlByChrome { if(cityEles){ for(def e : cityEles) { def href = e.attr("href") - if (!href.contains("fang.lianjia") && citys.contains(e.text())){ + if (!href.contains("fang.lianjia")){ cityMap.put(e.text(), href) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index e76898c7f..5c602f855 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -40,11 +40,11 @@ class DataStat { def run(){ - def source = "安居客"// - def citys = findCrawlCity(source, 202208); + def source = "我爱我家"// + def citys = findCrawlCity(source, 202209); for(def city : citys){ - def count1 = findCount(source, city, 202208); - def count2 = findCount(source, city, 202209); + def count1 = findCount(source, city, 202209); + def count2 = findCount(source, city, 202210); println city +" "+count1+" "+count2 // println count1 // println "--- ----" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 57c47a013..78f841680 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -24,8 +24,9 @@ class SynchDistrictPrice extends BaseBatch{ // "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"] def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"] - // "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产" - def esSources = [ "搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产"] + // "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田", + // "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士" + def esSources = ["贝壳找房":"贝壳找房","链家":"链家","麦田":"麦田"] static main(args) { From df0ee42df81a3a72e4fc4f4202fbfc083fdc89dd Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 17 Nov 2022 09:52:17 +0800 Subject: [PATCH 049/112] update --- .../ccbevaluation/ronghe/NoFenCity.groovy | 747 ++++++++++++++++++ .../thirdPriceCrawl/DataStandardBatch.groovy | 2 +- .../script/thirdPriceCrawl/DataStat.groovy | 2 +- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 4 +- 4 files changed, 751 insertions(+), 4 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/NoFenCity.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/NoFenCity.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/NoFenCity.groovy new file mode 100644 index 000000000..11270dab3 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/NoFenCity.groovy @@ -0,0 +1,747 @@ +package com.fangjia.script.ccbevaluation.ronghe + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.ccbevaluation.CcbFtpTestUtils + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import groovy.sql.Sql + +import java.sql.ResultSet +import java.sql.ResultSetMetaData + +@BatchOwner(Watcher.GaoJuan) +class ExportDistrictRongHeTest extends BaseBatch{ + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + static main(args) { + runBatch(new ExportDistrictRongHeTest(), args); + } + + @Override + void process(Object args) { + def curDate = 20220310//DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") + def fileName = "/ccb_rh/${curDate}" +// if(CommonDatautils.isWindow()){ +// fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}" +// } + CcbFtpTestUtils.delete(fileName) + + exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd")) + + exportBigdataCommunityGJRH() + exportBigdataBuildingFJW() + exportBigdataCommfeeGj() + + exportBigdataCommbuildstruGJ() + exportBigdataCommhousetypeGJ() + exportBigdataCommbuildyearGj() + exportBigdataCommhouseuseGJ() + exportBigdataPositionGJ() + exportBigdataDistrictFJW() + exportBigdataAliasFJW() + + exportBigdataBuildprojFjw() + exportBigdataCommlayoutpicFjw() + exportEvaluateRate() + + SSHHelper jschCommand = new SSHHelper(); + System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); + + exportOk(curDate) + + } + +// +// def runByCity(){ +// def citys = CcbCommonUtils.citys; +// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +// +// +//// //检查ok文件是否上传 +//// exportAllBigdataDealFJW() +//// for(def city : citys) { +//// def cityCode = cityCodeDataMap.get(city) +//// +// //按天导(一个文件夹下) +// //exportBigdataDealFJW(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd")) +// //按天导(每天一个文件夹) +// exportBigdataDealFJW2(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd")) +// +//// exportBigdataCommunityGJRH(cityCode) +//// exportBigdataBuildingFJW(cityCode) +//// exportBigdataCommfeeGj(cityCode) +//// +//// exportBigdataCommbuildstruGJ(cityCode) +//// exportBigdataCommhousetypeGJ(cityCode) +//// exportBigdataCommbuildyearGj(cityCode) +//// exportBigdataCommhouseuseGJ(cityCode) +//// exportBigdataPositionGJ(cityCode) +//// exportBigdataDistrictFJW(cityCode) +//// exportBigdataAliasFJW(cityCode) +//// +//// exportBigdataBuildprojFjw(cityCode) +//// exportBigdataCommlayoutpicFjw(cityCode) +//// } +//// exportEvaluateRate() +// +//// +// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") +// SSHHelper jschCommand = new SSHHelper(); +// System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); +// +// exportOk() +// } + + def exportBigdataCommunityGJRH(cityCode){ + def sql = "select CITYID,DISTID,COMMID,COMMNAME,ADDRESS,LANDAREA,BUILDAREA,GREENINGRATE,PLOTRATIO,PARKINGCNT,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE,COMMALIAS,STREET,STREETNO,IFNEW,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,SOURCECODE from BIGDATA_COMMUNITY_FJW";// where CITYID='${cityCode}' + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.COMMNAME + data << r.COMMALIAS + data << r.DISTID + data << r.STREET + data << r.STREETNO + data << r.ADDRESS + data << r.LANDAREA + data << r.BUILDAREA + data << r.COMMTYPE + data << r.IFNEW + data << r.GREENINGRATE + data << r.PLOTRATIO + data << r.PARKINGCNT + data << '' + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + println "BIGDATA_COMMUNITY_FJW 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_COMMUNITY_FJW 上传Ftp完成......" + } + def exportBigdataCommbuildstruGJ(){ + def sql = "select CITYID,COMMID,BUILDSTRU,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDSTRU_FJW";// where CITYID='${cityCode}' + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.BUILDSTRU + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_COMMBUILDSTRU_FJW 上传Ftp完成......" + } + + def exportBigdataCommhousetypeGJ(cityCode){ + def sql = "select CITYID,COMMID,HOUSETYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSETYPE_FJW";// where CITYID='${cityCode}' + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.HOUSETYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_COMMHOUSETYPE_FJW 上传Ftp完成......" + } + + def exportBigdataCommbuildyearGj(){ + def sql = "select CITYID,COMMID,BUILDYEAR,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDYEAR_FJW";// where CITYID='${cityCode}' + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.BUILDYEAR + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh", fileName, curDate) + println "BIGDATA_COMMBUILDYEAR_FJW 上传Ftp完成......" + } + + def exportBigdataCommfeeGj(){ + def sql = "select CITYID,COMMID,COMPID,FEE,HOUSEUSE,BUILDTYPE,FEETYPE,NOTE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMFEE_FJW";//where CITYID='${cityCode}' + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.COMPID + data << r.FEE + data << r.HOUSEUSE + if(r.BUILDTYPE){ + def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.FEETYPE + data << r.NOTE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_COMMFEE_FJW 上传Ftp完成......" + } + + + def exportBigdataCommhouseuseGJ(){ + def sql = "select CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSEUSE_FJW";// where CITYID='${cityCode}' + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.HOUSEUSE + data << r.HOUSECNT + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_COMMHOUSEUSE_FJW 上传Ftp完成......" + } + + def exportBigdataPositionGJ(){ + def sql = "select CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_POSITION_FJW";// where CITYID='${cityCode}' + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.OBJECTTYPE + data << r.OBJECTID + data << r.POSITIONTYPE + data << r.SEQNO + data << r.LONGITUDE + data << r.LATITUDE + data << r.ALTITUDE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_POSITION_FJW 上传Ftp完成......" + } + + def exportBigdataDistrictFJW(){ + def sql = "select DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_DISTRICT_FJW";// where CITYID='${cityCode}' + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.DISTID + data << r.DISTNAME + data << r.DISTSPELL + data << r.CITYID + data << r.DISTCODE + data << r.SEQ + data << r.DISTTYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh", fileName, curDate) + println "BIGDATA_DISTRICT_FJW 上传Ftp完成......" + } + + def exportBigdataAliasFJW(){ + def sql = "select city_id, ca_id,address_id,address_name, status, exp_dt, version, is_key_name, data_source, description,alias_grade,addr_type_cd,ifexamine,ifdisplay from ADDRESS_ALIAS_FJW";// where city_id='${cityCode}' + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.city_id + data << r.ca_id + data << r.address_id + data << r.address_name + data << r.status + data << r.exp_dt + data << r.version + data << r.is_key_name + data << r.data_source + data << r.description + data << r.alias_grade + data << r.addr_type_cd + data << r.ifexamine + data << r.ifdisplay + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "ADDRESS_ALIAS_FJW 上传Ftp完成......" + } + + def exportBigdataDealFJW(dealdate){ + + def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd") + def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd") + def stop = false + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def index = 0 + + def curStartDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()),"yyyy-MM-dd") + def curSndDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(new Date()),"yyyy-MM-dd") + + while(!stop){ + index++ + def curStartStr = curStartDateStr + + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" + } + + def allRows = [] + if(index==1){ + def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.DEALID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.STREET + data << r.STREETNO + data << r.COMMID + data << r.COMMNAME + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.UNITID + data << r.UNITNO + data << r.UNITNAME + data << r.HOUSEID + data << r.ADDRESS + data << r.BUILDYEAR + data << r.PROPTYPE + data << r.PROPYEARS + data << r.PERIODNUM + data << r.FLOORNUM + data << r.FLOORTYPE + data << r.FLOORCNT + if(r.BUILDTYPE){ + def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.BUILDSTRU + data << r.BUILDAREA + data << r.LIVINGAREA + data << r.HOUSESTYLE + data << r.ROOMCNT + data << r.LIVINGROOMCNT + data << r.KITCHENCNT + data << r.BATHROOMCNT + data << r.HOUSEUSE + data << r.HOUSESTRU + data << r.HOUSEASPECT + data << r.FITMENT + data << r.EVALUATENO + data << r.DEALUNITPRICE + data << r.DEALTOTALPRICE + data << r.DECLAREPRICE + data << r.EVALUATEPRICE + data << r.CONTRACTNO + data << r.DEALDATE + data << r.DEALTYPE + data << r.PAYTYPE + data << r.DEALPERIOD + data << r.IFELEVATOR + data << r.COMPAREAGARDEN + data << r.COMPAREATERRACE + data << r.COMPAREABALCONY + data << r.COMPAREABAYWINDOW + data << r.COMPAREAATTIC + data << r.COMPAREAGARAGE + data << r.COMPAREABASEMENT + data << r.COMPAREAOTHER + data << r.DATASOURCE + data << r.AGENCYID + data << r.AGENCYNAME + data << r.REMARK + data << r.CREATETIME1 + data << r.UPDATETIME1 + data << r.SOURCECODE + data << r.WORKDATE + allRows << data + }); + } + if(curStartDateStr == curSndDateStr){ + stop = true + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh", fileName, curDate) + println "BIGDATA_DEAL_FJW 上传Ftp完成......${curStartStr}" + def curEndStr = DateUtils.add(curStartStr, 1) + curStartDateStr = curEndStr + } + } + + + def exportBigdataDealFJW2(curDate, dealdate){ + + def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd") + def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd") +// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + } + + def allRows = [] + def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.DEALID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.STREET + data << r.STREETNO + data << r.COMMID + data << r.COMMNAME + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.UNITID + data << r.UNITNO + data << r.UNITNAME + data << r.HOUSEID + data << r.ADDRESS + data << r.BUILDYEAR + data << r.PROPTYPE + data << r.PROPYEARS + data << r.PERIODNUM + data << r.FLOORNUM + data << r.FLOORTYPE + data << r.FLOORCNT + if(r.BUILDTYPE){ + def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.BUILDSTRU + data << r.BUILDAREA + data << r.LIVINGAREA + data << r.HOUSESTYLE + data << r.ROOMCNT + data << r.LIVINGROOMCNT + data << r.KITCHENCNT + data << r.BATHROOMCNT + data << r.HOUSEUSE + data << r.HOUSESTRU + data << r.HOUSEASPECT + data << r.FITMENT + data << r.EVALUATENO + data << r.DEALUNITPRICE + data << r.DEALTOTALPRICE + data << r.DECLAREPRICE + data << r.EVALUATEPRICE + data << r.CONTRACTNO + data << r.DEALDATE + data << r.DEALTYPE + data << r.PAYTYPE + data << r.DEALPERIOD + data << r.IFELEVATOR + data << r.COMPAREAGARDEN + data << r.COMPAREATERRACE + data << r.COMPAREABALCONY + data << r.COMPAREABAYWINDOW + data << r.COMPAREAATTIC + data << r.COMPAREAGARAGE + data << r.COMPAREABASEMENT + data << r.COMPAREAOTHER + data << r.DATASOURCE + data << r.AGENCYID + data << r.AGENCYNAME + data << r.REMARK + data << r.CREATETIME1 + data << r.UPDATETIME1 + data << r.SOURCECODE + data << r.WORKDATE + allRows << data + }); + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_DEAL_FJW 上传Ftp完成......${endDateStr}" + } + + def exportAllBigdataDealFJW(){ + def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where DEALDATE + def data = [] + data << r.CITYID + data << r.DEALID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.STREET + data << r.STREETNO + data << r.COMMID + data << r.COMMNAME + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.UNITID + data << r.UNITNO + data << r.UNITNAME + data << r.HOUSEID + data << r.ADDRESS + data << r.BUILDYEAR + data << r.PROPTYPE + data << r.PROPYEARS + data << r.PERIODNUM + data << r.FLOORNUM + data << r.FLOORTYPE + data << r.FLOORCNT + if(r.BUILDTYPE){ + def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.BUILDSTRU + data << r.BUILDAREA + data << r.LIVINGAREA + data << r.HOUSESTYLE + data << r.ROOMCNT + data << r.LIVINGROOMCNT + data << r.KITCHENCNT + data << r.BATHROOMCNT + data << r.HOUSEUSE + data << r.HOUSESTRU + data << r.HOUSEASPECT + data << r.FITMENT + data << r.EVALUATENO + data << r.DEALUNITPRICE + data << r.DEALTOTALPRICE + data << r.DECLAREPRICE + data << r.EVALUATEPRICE + data << r.CONTRACTNO + data << r.DEALDATE + data << r.DEALTYPE + data << r.PAYTYPE + data << r.DEALPERIOD + data << r.IFELEVATOR + data << r.COMPAREAGARDEN + data << r.COMPAREATERRACE + data << r.COMPAREABALCONY + data << r.COMPAREABAYWINDOW + data << r.COMPAREAATTIC + data << r.COMPAREAGARAGE + data << r.COMPAREABASEMENT + data << r.COMPAREAOTHER + data << r.DATASOURCE + data << r.AGENCYID + data << r.AGENCYNAME + data << r.REMARK + data << r.CREATETIME1 + data << r.UPDATETIME1 + data << r.SOURCECODE + data << r.WORKDATE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + println "BIGDATA_DEAL_FJW 导出完成......" + } + + + def exportBigdataBuildingFJW(cityCode){ + def sql = "select CITYID, BUILDID, BUILDNO, BUILDNAME, BUILDALIAS, DISTID, COMMID, BUILDYEAR, BUILDTYPE, BUILDSTRU, FLOORCNT, UNITCNT, HOUSECNT, IFELEVATOR, ELEVATORCNT, ELEVATORHOUSE, REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, VALID_FLAG from BIGDATA_BUILDING_FJW";//where CITYID='${cityCode}' + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.BUILDALIAS + data << r.DISTID + data << r.COMMID + data << r.BUILDYEAR + if(r.BUILDTYPE){ + def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.BUILDSTRU + data << r.FLOORCNT + data << r.UNITCNT + data << r.HOUSECNT + data << r.IFELEVATOR + data << r.ELEVATORCNT + data << r.ELEVATORHOUSE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.VALID_FLAG + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_BUILDING_FJW 上传Ftp完成......" + } + + + + def exportEvaluateRate(){ + def sql = "select ID, RATE_TYPE, RATE, RANK, CITY, REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from EVALUATE_RATE_FJW"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.RATE_TYPE + data << r.RATE + data << r.RANK + data << r.CITY + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "EVALUATE_RATE_FJW 上传Ftp完成......" + } + + def exportBigdataBuildprojFjw(){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" + } + FileUtils.batchCreateFile([], fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_BUILDPROJ_FJW 上传Ftp完成......" + } + + def exportBigdataCommlayoutpicFjw(){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" + } + FileUtils.batchCreateFile([], fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......" + } + + def exportOk(curDate){ +// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok" + if(CommonDatautils.isWindow()){ + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok" + } + FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",dataCodeFileName, curDate) + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index 1ae98d64a..b5f86bf1b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -920,7 +920,7 @@ class DataStandardBatch extends BaseBatch{ def findAllNoMatchData(){ DBObject qdata = new BasicDBObject(); qdata.put("s_date", SHORT_DATE) - qdata.put("city","福州") +// qdata.put("city","福州") // qdata.put("source","安居客") // qdata.put("name",Pattern.compile("龚家宅.*")) qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false)); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index 5c602f855..78d3fceac 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -40,7 +40,7 @@ class DataStat { def run(){ - def source = "我爱我家"// + def source = "链家"// def citys = findCrawlCity(source, 202209); for(def city : citys){ def count1 = findCount(source, city, 202209); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 78f841680..a282f1c37 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -26,7 +26,7 @@ class SynchDistrictPrice extends BaseBatch{ // "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田", // "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士" - def esSources = ["贝壳找房":"贝壳找房","链家":"链家","麦田":"麦田"] + def esSources = ["麦田":"麦田","链家":"链家"] static main(args) { @@ -42,7 +42,7 @@ class SynchDistrictPrice extends BaseBatch{ println "starting......" def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd") - def citys = CommonUtils.findAllCity() + def citys = ["福州"]//CommonUtils.findAllCity() esSources.each{esSource, mgSource-> for(def city : citys){ From 96f4a06c1df14cd2a5210e667cf88d2c3b59f123 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 18 Nov 2022 10:07:40 +0800 Subject: [PATCH 050/112] update --- .../script/ccbevaluation/BigdataDealBatch.groovy | 4 ++-- .../script/ccbevaluation/BigdataForsaleBatch.groovy | 4 ++-- .../fangjia/script/ccbevaluation/CcbCommonUtils.groovy | 2 +- .../fangjia/script/ccbevaluation/TradeStatLog.groovy | 10 +++++----- .../script/thirdPriceCrawl/SynchDistrictPrice.groovy | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 00771ac49..86b64e4db 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -33,8 +33,8 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataDealBatch extends BaseBatch{ - def startDate = 202206 - def endDate = 202208 + def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")); + def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 73b4cf4b0..1c5031090 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -74,8 +74,8 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); -// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220918, 20221019); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20221019); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20221018, 20221119); +// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20221019); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); //测试通过匹配接口配房源效果 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index c126ea1ab..d9d7660c0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -42,7 +42,7 @@ class CcbCommonUtils { public static DSCollection dataStoreConnection = null; - public static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", + public static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index c2f488b5f..fd96c4d80 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -75,13 +75,13 @@ class TradeStatLog { def run(){ def citys = CcbCommonUtils.citys_92; def rets = [] - rets << ["城市","7月总量","8月总量","9月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] + rets << ["城市","8月总量","9月总量","10月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"] //def startDate = "2021-05-01" - def startDate = "2022-07-01" - def date = "2022-08-01" - def endDate = "2022-09-01" - def newDate = "2022-10-01" + def startDate = "2022-08-01" + def date = "2022-09-01" + def endDate = "2022-10-01" + def newDate = "2022-11-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index a282f1c37..f49a13675 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -26,7 +26,7 @@ class SynchDistrictPrice extends BaseBatch{ // "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田", // "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士" - def esSources = ["麦田":"麦田","链家":"链家"] + def esSources = ["链家":"链家"] static main(args) { @@ -42,7 +42,7 @@ class SynchDistrictPrice extends BaseBatch{ println "starting......" def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd") - def citys = ["福州"]//CommonUtils.findAllCity() + def citys = CommonUtils.findAllCity() esSources.each{esSource, mgSource-> for(def city : citys){ From 364ef055982fe0201af54372b12071380c9b25fe Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 18 Nov 2022 11:14:21 +0800 Subject: [PATCH 051/112] =?UTF-8?q?=E6=98=BE=E7=A4=BA=E5=B0=8F=E6=95=B0?= =?UTF-8?q?=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/fangjia/core/util/DataUtil.java | 2 +- .../weixin/WeiXinEvaluationController.java | 62 +++++++++++++++---- .../open/model/weixin/EvaluationModel.java | 6 +- .../com/fangjia/open/util/BHBankUtils.java | 2 +- .../main/webapp/weiXinEvaluation/bhform.ftl | 19 +++--- .../main/webapp/weiXinEvaluation/bhreport.ftl | 8 +-- 6 files changed, 72 insertions(+), 27 deletions(-) diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java b/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java index 608508eaa..24e130e93 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java @@ -1845,7 +1845,7 @@ public class DataUtil { // // System.out.println(FourUpSixInto("258.393", 2)); - String href = encodePicUrl("636b079c303c3f7093fd952a", "o", "upload"); + String href = encodePicUrl("63628191303c3f2682f4fbe6", "o", "upload"); String link = "http://static.fangjia.com/stc/fs/"+href; System.out.println(link); } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java index ed8766b8c..e8cf6be57 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java @@ -232,13 +232,39 @@ public class WeiXinEvaluationController { jsonObject.put("rn", fjtEvaluation.getRoomNum()); jsonObject.put("wylx", StringUtils.isBlank(BUILDTYPE_MAP.get(fjtEvaluation.getBuildType())) ? "-" : BUILDTYPE_MAP.get(fjtEvaluation.getBuildType()));//物业类型 jsonObject.put("mj", fjtEvaluation.getHouseArea());//面积 - String zj = "-"; if(evaluationAudit != null && evaluationAudit.getInquirePrice() > 0){ - zj = String.valueOf((int)(evaluationAudit.getInquirePrice() / 10000)); + double zj = DataUtil.getDoubleRound(2, evaluationAudit.getInquirePrice() / 10000.0); + if(zj == (int)zj){ + jsonObject.put("zj", (int)zj);//总价 + }else{ + jsonObject.put("zj", zj);//总价 + } + }else{ + jsonObject.put("zj", "-");//总价 } - jsonObject.put("zj", zj);//总价 - jsonObject.put("ckj", (evaluationAudit == null || evaluationAudit.getHopePrice() == null) ? "-" : (int)(evaluationAudit.getHopePrice()/ 10000));//期望价 - jsonObject.put("zsj", (evaluationAudit == null || evaluationAudit.getLastPrice() == null) ? "-" : (int)(evaluationAudit.getLastPrice()/ 10000));//终审价 + + if(evaluationAudit == null || evaluationAudit.getHopePrice() == null){ + jsonObject.put("ckj","-"); + }else{ + double ckj = DataUtil.getDoubleRound(2,evaluationAudit.getHopePrice()/10000.0); + if(ckj == (int)ckj){ + jsonObject.put("ckj", (int)ckj);//期望价 + }else{ + jsonObject.put("ckj", ckj);//期望价 + } + } + + if(evaluationAudit == null || evaluationAudit.getLastPrice() == null){ + jsonObject.put("zsj","-"); + }else{ + double zsj = DataUtil.getDoubleRound(2,evaluationAudit.getLastPrice()/ 10000.0); + if(zsj == (int)zsj){ + jsonObject.put("zsj", (int)zsj);//终审价 + }else{ + jsonObject.put("zsj", zsj);//终审价 + } + } + jsonObject.put("zdr", StringUtils.isBlank(fjtEvaluation.getLender()) ? "-" : fjtEvaluation.getLender());//主贷人 if(evaluationAudit==null || (evaluationAudit!=null && StringUtils.isBlank(evaluationAudit.getInquireRemark()))){ jsonObject.put("pgsm","");//评估说明 @@ -258,7 +284,13 @@ public class WeiXinEvaluationController { if(totalPrice>0){ evaluationAudit.setAuditStatus(1); evaluationAudit.setInquirePrice(totalPrice); - jsonObject.put("zj", (int)(totalPrice/10000)); + + double zj = DataUtil.getDoubleRound(2, totalPrice / 10000.0); + if(zj == (int)zj){ + jsonObject.put("zj", (int)zj);//总价 + }else{ + jsonObject.put("zj", zj);//总价 + } bohaiEvaluationAuditService.updateBohaiEvaluationAudit(evaluationAudit); jsonObject.put("shzt", "价格已审核"); } @@ -288,7 +320,12 @@ public class WeiXinEvaluationController { evaluationAudit.setFeedbackMsg(callMsg.toString()); } - jsonObject.put("zsj",(int)(inquirePrice/ 10000));//终审价 + double zsj = DataUtil.getDoubleRound(2,inquirePrice/ 10000.0);//终审价 + if(zsj == (int)zsj){ + jsonObject.put("zsj", (int)zsj);//终审价 + }else{ + jsonObject.put("zsj", zsj);//终审价 + } bohaiEvaluationAuditService.updateBohaiEvaluationAudit(evaluationAudit); jsonObject.put("shzt", "审核完成"); } @@ -407,6 +444,9 @@ public class WeiXinEvaluationController { public Object businessUserEvaluateAction(EvaluationModel model, HttpServletRequest request){ + JsonModel json = new JsonModel(); + Map data = new HashMap(); + District district = districtService.findById(model.getDistrict().getId()); FjtEvaluation evaluation = model.getFjtEvaluation(); if(evaluation.getHouseArea() <=0 || evaluation.getHouseHight() <= 0 @@ -511,7 +551,7 @@ public class WeiXinEvaluationController { } System.out.println("渤海询价搜索 "+evaluation.getAddress()); JSONObject bhApiResult = BHBankUtils.getInquirePrice(model, bhMatchMap, usersDetail, evaluation.getTotalPrice()); - if(bhApiResult!=null){ + if(bhApiResult!=null && (Integer)bhApiResult.get("errCode")==0){ System.out.println("渤海询价搜索结果 "+bhApiResult.toString()); JSONObject dataObj = (JSONObject) bhApiResult.get("data"); String queryId = (String) dataObj.get("queryId"); @@ -540,16 +580,16 @@ public class WeiXinEvaluationController { audit.setDateCreated(new Date()); audit.setDateCreated(new Date()); bohaiEvaluationAuditService.saveBohaiEvaluationAudit(audit); + }else{ + data.put("msg", "评估失败!"); } }else{ fjtEvaluationService.saveEvaluation(evaluation); System.out.println("保存评估记录成功 "+evaluation.getDistrictName()); } - Map data = new HashMap(); data.put("id", String.valueOf(evaluation.getId())); data.put("role", String.valueOf(fjtUser.getRole())); - JsonModel json = new JsonModel(); json.setJsonModel(data); return json; } @@ -891,7 +931,7 @@ public class WeiXinEvaluationController { if(StringUtils.isNotBlank(model.getEvaluationId()) && model.getHopePrice()!=null){ BohaiEvaluationAudit evaluationAudit = bohaiEvaluationAuditService.findByBohaiEvaluationAuditId(Long.valueOf(model.getEvaluationId())); if(evaluationAudit != null){ - evaluationAudit.setHopePrice(model.getHopePrice()*10000); + evaluationAudit.setHopePrice((int)(model.getHopePrice()*10000)); evaluationAudit.setLastUpdated(new Date()); evaluationAudit.setAuditStatus(2); diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/weixin/EvaluationModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/weixin/EvaluationModel.java index 20f093053..cdf645a31 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/weixin/EvaluationModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/weixin/EvaluationModel.java @@ -43,7 +43,7 @@ public class EvaluationModel extends BaseModel{ private Integer role; private String cardId; private Map loanInfoMap; - private Integer hopePrice; + private Double hopePrice; private String remark; private Integer loan; // private String preCode;//渤海接口--楼盘编码 @@ -222,11 +222,11 @@ public class EvaluationModel extends BaseModel{ this.bohaiEvaluationAudit = bohaiEvaluationAudit; } - public Integer getHopePrice() { + public Double getHopePrice() { return hopePrice; } - public void setHopePrice(Integer hopePrice) { + public void setHopePrice(Double hopePrice) { this.hopePrice = hopePrice; } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java b/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java index 6a861a303..1168d8bd5 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java @@ -210,7 +210,7 @@ public class BHBankUtils { return null; } - public static String inquireFeedback(String name, String queryId, Integer hopePrice){ + public static String inquireFeedback(String name, String queryId, Double hopePrice){ String feedbackId = null; List list = new ArrayList(); Map params = new HashMap(); diff --git a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhform.ftl b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhform.ftl index 8a39228e1..68519a79b 100644 --- a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhform.ftl +++ b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhform.ftl @@ -252,16 +252,21 @@ async: false, method: "POST", success: function (data) { - if (data.id == undefined) { + if(data.msg){ $.toast("浼颁环澶辫触锛岃閲嶆柊璇勪及"); - $("#evaluateBtn").prop("disabled", false) - } else { - if(data.role == "17"){ - location.href = "bhreport?evaluationId=" + data.id - }else{ - location.href = "report?evaluationId=" + data.id + }else{ + if (data.id == undefined) { + $.toast("浼颁环澶辫触锛岃閲嶆柊璇勪及"); + $("#evaluateBtn").prop("disabled", false) + } else { + if(data.role == "17"){ + location.href = "bhreport?evaluationId=" + data.id + }else{ + location.href = "report?evaluationId=" + data.id + } } } + }, error: function(XMLHttpRequest){ $.toast("鏈嶅姟鍣ㄥ紓甯革紝璇烽噸鏂拌瘎浼�"); diff --git a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl index ecf07ff92..6c6e97d28 100644 --- a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl +++ b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl @@ -38,7 +38,7 @@

*璇风‘璁や笌浜ц瘉鍦板潃涓�鑷�

<#if bohaiEvaluationAudit?? && bohaiEvaluationAudit.auditStatus!=0 && bohaiEvaluationAudit.inquirePrice gt 0> -

璇勪及鎬讳环锛�<#if fjtEvaluation??><#if bohaiEvaluationAudit.lastPrice??>${(bohaiEvaluationAudit.lastPrice/10000)?int}<#elseif bohaiEvaluationAudit.inquirePrice??>${(bohaiEvaluationAudit.inquirePrice/10000)?int} 涓囧厓

+

璇勪及鎬讳环锛�<#if fjtEvaluation??><#if bohaiEvaluationAudit.lastPrice??>${(bohaiEvaluationAudit.lastPrice/10000)?string('0.##')}<#elseif bohaiEvaluationAudit.inquirePrice??>${(bohaiEvaluationAudit.inquirePrice/10000)?string('0.##')} 涓囧厓

璇勪及鍗曚环锛�<#if bohaiEvaluationAudit.lastPrice??>${(bohaiEvaluationAudit.lastPrice/fjtEvaluation.houseArea)?int}<#elseif bohaiEvaluationAudit.inquirePrice??>${(bohaiEvaluationAudit.inquirePrice/fjtEvaluation.houseArea)?int} 鍏�/骞崇背

<#if role?? && role==17 && loanInfoMap??>

鎵�灞炲垎绫伙細${loanInfoMap.type!}绫�

@@ -71,15 +71,15 @@ <#if bohaiEvaluationAudit?? && bohaiEvaluationAudit.lastPrice??>
- 瀹℃牳浠凤細${(bohaiEvaluationAudit.lastPrice!/10000)?int} 涓囧厓
- 鏈熸湜浠凤細${(bohaiEvaluationAudit.hopePrice!/10000)?int} 涓囧厓 + 瀹℃牳浠凤細${(bohaiEvaluationAudit.lastPrice!/10000)?string('0.##')} 涓囧厓
+ 鏈熸湜浠凤細${(bohaiEvaluationAudit.hopePrice!/10000)?string('0.##')} 涓囧厓
<#else> <#if bohaiEvaluationAudit?? && bohaiEvaluationAudit.hopePrice??>
- 鏈熸湜浠凤細${(bohaiEvaluationAudit.hopePrice!/10000)?int} 涓囧厓 + 鏈熸湜浠凤細${(bohaiEvaluationAudit.hopePrice!/10000)?string('0.##')} 涓囧厓 鍒锋柊
From 81af8ccac82850eebc699fa78e16c93721b6bd53 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 18 Nov 2022 16:23:58 +0800 Subject: [PATCH 052/112] =?UTF-8?q?=E7=AC=AC=E4=B8=89=E6=96=B9=E6=97=B6?= =?UTF-8?q?=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/ThirdPartyPriceBatch.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 7637891ea..8c12e22b0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202204//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202202 - def endDate = 202204 + def sDate= 202210//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202208 + def endDate = 202210 // def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") From a4346b666d1346366a536ef44a7845733d10cc88 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 18 Nov 2022 18:20:20 +0800 Subject: [PATCH 053/112] update --- .../controllers/weixin/WeiXinEvaluationController.java | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java index e8cf6be57..ba7849763 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java @@ -242,7 +242,6 @@ public class WeiXinEvaluationController { }else{ jsonObject.put("zj", "-");//总价 } - if(evaluationAudit == null || evaluationAudit.getHopePrice() == null){ jsonObject.put("ckj","-"); }else{ @@ -264,7 +263,6 @@ public class WeiXinEvaluationController { jsonObject.put("zsj", zsj);//终审价 } } - jsonObject.put("zdr", StringUtils.isBlank(fjtEvaluation.getLender()) ? "-" : fjtEvaluation.getLender());//主贷人 if(evaluationAudit==null || (evaluationAudit!=null && StringUtils.isBlank(evaluationAudit.getInquireRemark()))){ jsonObject.put("pgsm","");//评估说明 @@ -284,7 +282,6 @@ public class WeiXinEvaluationController { if(totalPrice>0){ evaluationAudit.setAuditStatus(1); evaluationAudit.setInquirePrice(totalPrice); - double zj = DataUtil.getDoubleRound(2, totalPrice / 10000.0); if(zj == (int)zj){ jsonObject.put("zj", (int)zj);//总价 @@ -315,11 +312,7 @@ public class WeiXinEvaluationController { evaluationAudit.setAuditStatus(3); evaluationAudit.setLastPrice(inquirePrice); evaluationAudit.setFeedbackStatus(feedbackStatus); - Object callMsg = object.get("feedbackCallMsg"); - if(callMsg!=null && !callMsg.toString().equals("null")){ - evaluationAudit.setFeedbackMsg(callMsg.toString()); - } - + evaluationAudit.setFeedbackMsg((String)object.get("feedbackCallMsg")); double zsj = DataUtil.getDoubleRound(2,inquirePrice/ 10000.0);//终审价 if(zsj == (int)zsj){ jsonObject.put("zsj", (int)zsj);//终审价 From 415a109a0e6bd752220c4745722c7decaa77c8d7 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 21 Nov 2022 10:20:51 +0800 Subject: [PATCH 054/112] =?UTF-8?q?=E7=99=BE=E7=9B=9B=E8=BF=94=E5=9B=9E1?= =?UTF-8?q?=EF=BC=8C=E5=9B=A0=E9=94=99=E8=AF=AF=E4=BF=A1=E6=81=AF=E5=A4=AA?= =?UTF-8?q?=E9=95=BF=EF=BC=8C=E8=BF=98=E6=98=AF=E9=BB=98=E8=AE=A4=E6=98=BE?= =?UTF-8?q?=E7=A4=BA=E6=88=91=E4=BB=AC=E7=9A=84=E6=8F=90=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../weixin/WeiXinEvaluationController.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java index ba7849763..0c8abd103 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java @@ -574,7 +574,16 @@ public class WeiXinEvaluationController { audit.setDateCreated(new Date()); bohaiEvaluationAuditService.saveBohaiEvaluationAudit(audit); }else{ - data.put("msg", "评估失败!"); + if(bhApiResult!=null && (Integer)bhApiResult.get("errCode")==1){ + JSONObject dataObj = (JSONObject) bhApiResult.get("data"); + if(StringUtils.isNotBlank((String)dataObj.get("remark"))){ + data.put("msg", (String)dataObj.get("remark")); + }else{ + data.put("msg", "估价失败,请重新评估!"); + } + }else{ + data.put("msg", "估价失败,请重新评估!"); + } } }else{ fjtEvaluationService.saveEvaluation(evaluation); From c7e417b17a6a6c005f9a4f6dc2ef3df988ea8627 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 21 Nov 2022 14:07:37 +0800 Subject: [PATCH 055/112] =?UTF-8?q?=E5=AF=B9=E4=BA=8E=E7=99=BE=E7=9B=9B?= =?UTF-8?q?=E8=BF=94=E5=9B=9EJSONNULL=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/weixin/WeiXinEvaluationController.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java index 0c8abd103..c24973754 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java @@ -312,7 +312,10 @@ public class WeiXinEvaluationController { evaluationAudit.setAuditStatus(3); evaluationAudit.setLastPrice(inquirePrice); evaluationAudit.setFeedbackStatus(feedbackStatus); - evaluationAudit.setFeedbackMsg((String)object.get("feedbackCallMsg")); + Object callMsg = object.get("feedbackCallMsg"); + if(!callMsg.toString().equals("null")){ + evaluationAudit.setFeedbackMsg((String)object.get("feedbackCallMsg")); + } double zsj = DataUtil.getDoubleRound(2,inquirePrice/ 10000.0);//终审价 if(zsj == (int)zsj){ jsonObject.put("zsj", (int)zsj);//终审价 From e8d12518705091f1749d85aee1b7b5cfdf75889d Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 21 Nov 2022 17:54:00 +0800 Subject: [PATCH 056/112] update --- .../ccbevaluation/BigdataBuildingBatch.groovy | 2 +- .../ccbevaluation/CcbCommonUtils.groovy | 5 ++ .../ccbevaluation/GuiDingPriceBatch.groovy | 7 ++- .../fangjia/script/ccbevaluation/Test.groovy | 60 +++++++++---------- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 2 +- .../ronghe/BigdataCommunityGJRHBatch.groovy | 10 +++- .../gj/paimai/BeiKeCrawlByChrome.groovy | 2 +- .../gj/paimai/LianjiaCrawlByChrome.groovy | 2 +- 8 files changed, 52 insertions(+), 38 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy index 7500406cc..431972b91 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy @@ -36,7 +36,7 @@ class BigdataBuildingBatch extends BaseBatch{ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, buildingMap) println city + " " + rets.size() - def fileName = "/home/goojia/ccb/bigdata_building_fjw.dat" + def fileName = "/ccb/bigdata_building_fjw.dat" if(CommonDatautils.isWindow()){ fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_building_fjw.dat" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index d9d7660c0..909ea8025 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -562,4 +562,9 @@ class CcbCommonUtils { return maxSeqno } + public static def deleteGuidedPrice(def dealdate){ + def sql = "delete from guided_price_gj where dealdate=to_date('${dealdate}','yyyy-mm-dd')" + sqlInstance.executeUpdate(sql) + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 66e8081dd..bd64cc3f8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -30,6 +30,8 @@ class GuiDingPriceBatch extends BaseBatch{ def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); def static final DBCollection dataProcessColl = CrawalDBUtil.getDataProcessCol(); + def curDate= DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-15"; + static main(args) { runBatch(new GuiDingPriceBatch(), args); } @@ -39,12 +41,13 @@ class GuiDingPriceBatch extends BaseBatch{ def citys = CcbCommonUtils.getRunningCitys(args) def rows = [] + CcbCommonUtils.deleteGuidedPrice(curDate) for (def city : citys) { def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-11") + def prices = CcbCommonUtils.findPersonPrice(city, "2022-10-27", "2022-11-18") if(prices){ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) @@ -150,7 +153,7 @@ class GuiDingPriceBatch extends BaseBatch{ ret << CcbCommonUtils.getCcbCity(dist.city) ret << areaData.district_code ret << areaData.district - ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss") + ret << curDate//DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss") ret << ps.guiding_value ret << "房价网" ret << "" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index f8e04a350..d36f08cc1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -447,23 +447,23 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business ////},0) // -//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\二期\\11.11\\删除小区(2).xlsx",{d-> -// if(d['A']){ -// def dbo = CommonDatautils.findDistrictById(d['C']) -// if(dbo){ -// def udata = new BasicDBObject() +ExcelUtils.multiThreadingModify("D:\\数据\\建行\\二期\\252个城市删除小区.xlsx",{d-> + if(d['A']){ + def dbo = CommonDatautils.findDistrictById(d['D']) + if(dbo){ + def udata = new BasicDBObject() // udata.put("name_cat", 11) -// udata.put("status", 1) -// udata.put("m_date",new Date()) -// -// def q = new BasicDBObject() -// q.put("_id", dbo._id) -// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) -// }else{ -// println dbo._id.toString() +" 没找到" -// } -// } -//},0) + udata.put("status", 1) + udata.put("m_date",new Date()) + + def q = new BasicDBObject() + q.put("_id", dbo._id) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + }else{ + println dbo._id.toString() +" 没找到" + } + } +},0) //def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"] //for(def c : citys){ @@ -471,20 +471,20 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business // println c +" "+districts.size() //} -def cityCodeMap = CcbCommonUtils.findCityCodeValue() -ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\252城第三方均价.xlsx", { d -> -// cityCodeMap.each {k,v-> -// if(d['A'].toString()==v){ -// d['C'] = k -// println k -// } -// } -// if(!d['D']){ -// d['D'] = findCount(d['A'].toString().toDouble().toInteger().toString(), sqlInstance94) -// } - d['F'] = findThirdCommCount(d['A'].toString().toDouble().toInteger().toString(), sqlInstance94) - -}); +//def cityCodeMap = CcbCommonUtils.findCityCodeValue() +//ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\252城第三方均价.xlsx", { d -> +//// cityCodeMap.each {k,v-> +//// if(d['A'].toString()==v){ +//// d['C'] = k +//// println k +//// } +//// } +//// if(!d['D']){ +//// d['D'] = findCount(d['A'].toString().toDouble().toInteger().toString(), sqlInstance94) +//// } +// d['F'] = findThirdCommCount(d['A'].toString().toDouble().toInteger().toString(), sqlInstance94) +// +//}); //def citys = CcbCommonUtils.citys_252 //def runCitys = ['聊城','宝鸡','韶关','阜阳','宜春','周口','河源','忻州','三门峡','永州','衢州','盘锦','白银','玉溪','达州','景德镇','鹤壁','辽阳','六盘水','张家界','海西州','兴安盟','阿坝州','那曲','西双版纳州','辽源','邯郸','平顶山','南阳','商丘','南平','黄冈','鄂尔多斯','泸州','六安','葫芦岛','潮州','宿州','庆阳','怀化','张掖','黔东南','云浮','本溪','百色','伊春','昭通','锡林郭勒盟','克拉玛依','天门','博尔塔拉','大兴安岭','潍坊','绵阳','梅州','襄阳','包头','枣庄','东营','信阳','晋中','开封','承德','茂名','荆门','红河','陇南','铁岭','黄山','咸宁','延安','广元','乌兰察布','随州','益阳','阳泉','内江','酒泉','毕节','安顺','丽江','商洛','来宾','黔西南','防城港','铜仁','儋州','崇左','衡阳','咸阳','新乡','南充','荆州','德阳','汉中','湘潭','赤峰','佳木斯','丹东','淮南','抚州','丽水','广安','亳州','昌吉','鄂州','白城','恩施','伊犁','铜陵','林芝','海东','桂林','临汾','渭南','滨州','菏泽','孝感','营口','濮阳','抚顺','自贡','新余','鸡西','钦州','淮北','安康','资阳','乌海','中卫','鹤岗','石嘴山','日喀则','昌都','德宏州','吐鲁番','阿勒泰','临沂','威海','泰安','宿迁','德州','安阳','驻马店','绥化','上饶','莆田','蚌埠','大庆','漯河','十堰','晋城','宣城','舟山','邵阳','大理','楚雄','天水','池州','松原','阜新','巴音郭楞','仙桃','武威','普洱','河池','铜川','迪庆州','潜江','贺州','塔城','济宁','江门','赣州','许昌','滁州','龙岩','运城','长治','乐山','郴州','遂宁','曲靖','呼伦贝尔','阳江','朝阳','巴彦淖尔','朔州','娄底','梧州','巴中','黔南','吴忠','临夏','凉山州','定西','萍乡','甘南','攀枝花','克孜勒苏','山南','临沧','金昌','邢台','衡水','宁德','榆林','鞍山','焦作','牡丹江','齐齐哈尔','眉山','三明','吕梁','吉安','平凉','通辽','揭阳','黄石','宜宾','延边','汕尾','玉林','通化','四平','阿克苏','贵港','白山','双鸭山','湘西','哈密','保山','鹰潭','喀什','雅安','黑河','和田','固原','阿拉善盟','七台河','嘉峪关'] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 8c12e22b0..03d4cfcb7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -67,7 +67,7 @@ class ThirdPartyPriceBatch extends BaseBatch { } def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] - FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true); + FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy index c5b66e238..770ea6914 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy @@ -32,8 +32,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", - "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + def sqlInstance = null public static def batchsize = 1000 static main(args) { @@ -43,6 +42,13 @@ class BigdataCommunityGJRHBatch extends BaseBatch { @Override public void process(Object args) { def citys = CcbCommonUtils.getRunningCitys(args) + if(citys.size()==92){ + sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + }else{ + sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + } def rows = [] def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() // for (def city : citys) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index f033823f9..ce6242a31 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -183,7 +183,7 @@ public class BeiKeCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern(startMonth + "-30", "yyyy-MM-dd") if(dealDate && dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e, region) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 0ca1bc62a..8e74425cb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -251,7 +251,7 @@ public class LianjiaCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern(startMonth + "-30", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e, regionText) From 55368a0a9fe0541d0c7410f69e1d0527dbfd2761 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 21 Nov 2022 17:55:15 +0800 Subject: [PATCH 057/112] update --- .../ccbevaluation/BigdataDealBatch.groovy | 59 ++++++++++++------- 1 file changed, 38 insertions(+), 21 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 86b64e4db..2803b3229 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -63,7 +63,7 @@ class BigdataDealBatch extends BaseBatch{ curDate = args[1].toString().toInteger() } - println "curDate:"+curDate +// println "curDate:"+curDate // for(def city : citys){ //// if(!ccbCitys.contains(city)){ // addDealPriceMatch(city); @@ -87,16 +87,16 @@ class BigdataDealBatch extends BaseBatch{ } - def trades = getCityData(city, statMap, ppeDists) - if(trades){ - def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") - rows.addAll(rets) - println "【${index}】"+city +" 成交共:"+rets.size() - - def cityDatas = [] - cityDatas << headers - cityDatas.addAll(rows) - } +// def trades = getCityData(city, statMap, ppeDists) +// if(trades){ +// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") +// rows.addAll(rets) +// println "【${index}】"+city +" 成交共:"+rets.size() +// +// def cityDatas = [] +// cityDatas << headers +// cityDatas.addAll(rows) +// } } @@ -105,7 +105,7 @@ class BigdataDealBatch extends BaseBatch{ fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" } - FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", true); + FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); println "txt 生成 " + rows.size() } @@ -114,9 +114,11 @@ class BigdataDealBatch extends BaseBatch{ def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(rets.size()) - def lianjiaPriceMap = findThirdPartyPrice("上海") - def startDate = DateUtils.parseFromTightByPattern(DateUtils.addMonth(new Date(),-2),'yyyy-MM-dd') +// def lianjiaPriceMap = findThirdPartyPrice("上海") +// def startDate = DateUtils.parseFromTightByPattern(DateUtils.addMonth(new Date(),-2),'yyyy-MM-dd') // def lianjiaPriceMap = getDistrictStatByCity("上海", startDate) + + def ccbPriceMap = getCommunityPrices("上海") for(def i=0;i Date: Mon, 21 Nov 2022 18:04:34 +0800 Subject: [PATCH 058/112] =?UTF-8?q?=E6=A0=B9=E6=8D=AE=E5=9F=8E=E5=B8=82?= =?UTF-8?q?=E9=87=8F=E8=8E=B7=E5=8F=96=E6=95=B0=E6=8D=AE=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/ccbevaluation/CcbCommonUtils.groovy | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 909ea8025..517f8c44c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -42,8 +42,7 @@ class CcbCommonUtils { public static DSCollection dataStoreConnection = null; - public static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", - "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + public static def sqlInstance = null public static def getRunningCitys(def args){ @@ -53,10 +52,16 @@ class CcbCommonUtils { def countStr = args[0] if(countStr=="344"){ citys = CcbCommonUtils.citys_all + sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") }else if(countStr=="92"){ citys = CcbCommonUtils.citys_92 + sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") }else if(countStr=="252"){ citys = CcbCommonUtils.citys_252 + sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") } }else{ citys = CcbCommonUtils.citys_92 From 2f8105cbb4b5bd63d9c05c9688a84b50ad391b73 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 21 Nov 2022 18:37:31 +0800 Subject: [PATCH 059/112] =?UTF-8?q?252=E5=9F=8E=E6=88=90=E4=BA=A4=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/BigdataDealBatch.groovy | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 2803b3229..28c543e60 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -87,16 +87,16 @@ class BigdataDealBatch extends BaseBatch{ } -// def trades = getCityData(city, statMap, ppeDists) -// if(trades){ -// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") -// rows.addAll(rets) -// println "【${index}】"+city +" 成交共:"+rets.size() -// -// def cityDatas = [] -// cityDatas << headers -// cityDatas.addAll(rows) -// } + def trades = getCityData(city, statMap, ppeDists) + if(trades){ + def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") + rows.addAll(rets) + println "【${index}】"+city +" 成交共:"+rets.size() + + def cityDatas = [] + cityDatas << headers + cityDatas.addAll(rows) + } } @@ -105,7 +105,7 @@ class BigdataDealBatch extends BaseBatch{ fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" } - FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); + FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", true); println "txt 生成 " + rows.size() } From c93aaeae37b8bd11c6cb339de750ac676f1fe893 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 22 Nov 2022 10:44:33 +0800 Subject: [PATCH 060/112] =?UTF-8?q?252=E5=9F=8E=E8=9E=8D=E5=90=88=E6=B5=8B?= =?UTF-8?q?=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ronghe/ExportDistrictRongHeTest.groovy | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy index 26520d0d5..28f2b0ffa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy @@ -28,36 +28,36 @@ class ExportDistrictRongHeTest extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //按天导(每天一个文件夹) - // exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) + exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) // if(curDate.toString().endsWith("10")){ -// exportBigdataBuildprojFjw() -// exportBigdataCommlayoutpicFjw() -// exportEvaluateRate() + exportBigdataBuildprojFjw() + exportBigdataCommlayoutpicFjw() + exportEvaluateRate() -// exportBigdataCommunityGJRH()// -// exportBigdataBuildingFJW()// -// exportBigdataCommfeeGj()// -// -// exportBigdataCommbuildstruGJ()// -// exportBigdataCommhousetypeGJ()// -// exportBigdataCommbuildyearGj()// -// exportBigdataCommhouseuseGJ()// -// exportBigdataPositionGJ()// -// exportBigdataDistrictFJW()// -// exportBigdataAliasFJW()// + exportBigdataCommunityGJRH()// + exportBigdataBuildingFJW()// + exportBigdataCommfeeGj()// - def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() - exportBigdataCityPrice(queryDate) - exportBigdataDistPrice(queryDate) - exportBigdataCommPrice(queryDate) + exportBigdataCommbuildstruGJ()// + exportBigdataCommhousetypeGJ()// + exportBigdataCommbuildyearGj()// + exportBigdataCommhouseuseGJ()// + exportBigdataPositionGJ()// + exportBigdataDistrictFJW()// + exportBigdataAliasFJW()// + +// def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() +// exportBigdataCityPrice(queryDate) +// exportBigdataDistPrice(queryDate) +// exportBigdataCommPrice(queryDate) // } - SSHHelper jschCommand = new SSHHelper(); - System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); - - exportOk(curDate) +// SSHHelper jschCommand = new SSHHelper(); +// System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); +// +// exportOk(curDate) } From 45631507d5c58178548d8ebcc85b21ba40fdc228 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 22 Nov 2022 10:54:25 +0800 Subject: [PATCH 061/112] 236-94 --- .../script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy index 28f2b0ffa..d91239c2a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy @@ -13,7 +13,7 @@ import groovy.sql.Sql @BatchOwner(Watcher.GaoJuan) class ExportDistrictRongHeTest extends BaseBatch{ - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") static main(args) { runBatch(new ExportDistrictRongHeTest(), args); From 74dfc11c3f4e7b775265bfebf8cb28125c3f945e Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 22 Nov 2022 11:05:29 +0800 Subject: [PATCH 062/112] =?UTF-8?q?=E8=A1=8C=E6=83=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ronghe/ExportDistrictRongHeTest.groovy | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy index d91239c2a..90a196d8c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy @@ -28,36 +28,36 @@ class ExportDistrictRongHeTest extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //按天导(每天一个文件夹) - exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) +// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) // if(curDate.toString().endsWith("10")){ - exportBigdataBuildprojFjw() - exportBigdataCommlayoutpicFjw() - exportEvaluateRate() +// exportBigdataBuildprojFjw() +// exportBigdataCommlayoutpicFjw() +// exportEvaluateRate() +// +// exportBigdataCommunityGJRH()// +// exportBigdataBuildingFJW()// +// exportBigdataCommfeeGj()// +// +// exportBigdataCommbuildstruGJ()// +// exportBigdataCommhousetypeGJ()// +// exportBigdataCommbuildyearGj()// +// exportBigdataCommhouseuseGJ()// +// exportBigdataPositionGJ()// +// exportBigdataDistrictFJW()// +// exportBigdataAliasFJW()// - exportBigdataCommunityGJRH()// - exportBigdataBuildingFJW()// - exportBigdataCommfeeGj()// - - exportBigdataCommbuildstruGJ()// - exportBigdataCommhousetypeGJ()// - exportBigdataCommbuildyearGj()// - exportBigdataCommhouseuseGJ()// - exportBigdataPositionGJ()// - exportBigdataDistrictFJW()// - exportBigdataAliasFJW()// - -// def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() -// exportBigdataCityPrice(queryDate) -// exportBigdataDistPrice(queryDate) -// exportBigdataCommPrice(queryDate) + def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() + exportBigdataCityPrice(queryDate) + exportBigdataDistPrice(queryDate) + exportBigdataCommPrice(queryDate) // } -// SSHHelper jschCommand = new SSHHelper(); -// System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); -// -// exportOk(curDate) + SSHHelper jschCommand = new SSHHelper(); + System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); + + exportOk(curDate) } From c100a1037595f7103b5a497efa5afa8cfa609fc5 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 22 Nov 2022 11:08:48 +0800 Subject: [PATCH 063/112] 236-94 --- .../script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy index 90a196d8c..ccb575126 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy @@ -13,7 +13,7 @@ import groovy.sql.Sql @BatchOwner(Watcher.GaoJuan) class ExportDistrictRongHeTest extends BaseBatch{ - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") static main(args) { runBatch(new ExportDistrictRongHeTest(), args); From b610026293f70c71d381f6582c5029f4d4cf0582 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 22 Nov 2022 11:29:23 +0800 Subject: [PATCH 064/112] =?UTF-8?q?252=E5=9F=8E=E6=88=BF=E6=BA=90=E6=97=B6?= =?UTF-8?q?=E9=97=B4=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy | 4 ++-- .../ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 1c5031090..d89e8501e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -74,8 +74,8 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20221018, 20221119); -// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20221019); +// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20221018, 20221119); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20221119); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); //测试通过匹配接口配房源效果 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy index ccb575126..161f82f00 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy @@ -54,7 +54,7 @@ class ExportDistrictRongHeTest extends BaseBatch{ exportBigdataCommPrice(queryDate) // } - SSHHelper jschCommand = new SSHHelper(); + SSHHelperTest jschCommand = new SSHHelperTest(); System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); exportOk(curDate) From 08f5cc83440c729bc2324113eb2c3377a56b3d0a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 23 Nov 2022 10:19:23 +0800 Subject: [PATCH 065/112] =?UTF-8?q?=E5=9B=A0=E7=99=BE=E7=9B=9B=E6=8A=8A?= =?UTF-8?q?=E8=AF=84=E4=BC=B0=E4=B8=8D=E5=87=BA=E6=9D=A5=E7=9A=84=E6=94=B9?= =?UTF-8?q?=E4=B8=BA=E6=AD=A3=E5=B8=B8=E7=8A=B6=E6=80=81=EF=BC=8C=E8=BF=94?= =?UTF-8?q?=E5=9B=9E=E4=BB=B7=E6=A0=BC=E4=B8=BA0=EF=BC=8C=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E7=8A=B6=E6=80=81=E7=A0=81=E5=8F=98=E4=B8=BA=E5=B7=B2?= =?UTF-8?q?=E5=AE=A1=E6=A0=B8=E7=8A=B6=E6=80=81=EF=BC=8C=E7=8E=B0=E5=8A=A0?= =?UTF-8?q?=E5=85=A5=E4=BB=B7=E6=A0=BC=E5=88=A4=E6=96=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/weixin/WeiXinEvaluationController.java | 4 ++-- .../src/main/java/com/fangjia/open/util/BHBankUtils.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java index c24973754..4be686b2c 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java @@ -552,11 +552,11 @@ public class WeiXinEvaluationController { JSONObject dataObj = (JSONObject) bhApiResult.get("data"); String queryId = (String) dataObj.get("queryId"); Integer inquirePrice = (Integer) dataObj.get("inquirePrice"); - Integer auditStatus = null; + Integer auditStatus = 0; String inquireRemark = (String) dataObj.get("remark"); if((Integer) bhApiResult.get("errCode") == 1){//待人工核验 auditStatus = 0; - }else if((Integer) bhApiResult.get("errCode") == 0){//正常返回 + }else if((Integer) bhApiResult.get("errCode") == 0 && inquirePrice>0){//正常返回 auditStatus = 1; } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java b/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java index 1168d8bd5..1b7652515 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java @@ -17,7 +17,7 @@ import java.util.*; public class BHBankUtils { -// public static String token = "bh";// 原:bohaifangjiatest +// public static String token = "bohaifangjia";// 原:bohaifangjiatest // public static String uri = "http://shbreac.ruitingcc.com/API/";//测试 原:http://topen.shbrea.com.cn/API From 3f5030d253811a98a3f541a9c111388d3b45ea57 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 24 Nov 2022 10:37:24 +0800 Subject: [PATCH 066/112] =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E7=85=A7=E7=89=87?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/com/fangjia/core/util/DataUtil.java | 2 +- .../com/fangjia/dal/sql/model/ReportFileAttach.java | 11 +++++++++++ .../weixin/WeiXinEvaluationController.java | 1 + 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java b/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java index 24e130e93..30aa6763d 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java @@ -1845,7 +1845,7 @@ public class DataUtil { // // System.out.println(FourUpSixInto("258.393", 2)); - String href = encodePicUrl("63628191303c3f2682f4fbe6", "o", "upload"); + String href = encodePicUrl("635240149c21300bba5184e1", "o", "upload"); String link = "http://static.fangjia.com/stc/fs/"+href; System.out.println(link); } diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/ReportFileAttach.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/ReportFileAttach.java index acc9fcd36..34eb3efd6 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/ReportFileAttach.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/ReportFileAttach.java @@ -3,6 +3,7 @@ package com.fangjia.dal.sql.model; import com.fangjia.dal.SqlEntity; import javax.persistence.*; +import java.util.Date; @Entity @Table(name = "report_file_attach") @@ -14,6 +15,7 @@ public class ReportFileAttach implements SqlEntity { private Integer fileClass;//1:询价单 2:产证 3:房屋照片 4:其它附件 5:产调报告或权籍 private String fileName; private String fileId; + private Date dateCreated; @Id @Column(name = "id") @@ -79,4 +81,13 @@ public class ReportFileAttach implements SqlEntity { public void setFileId(String fileId) { this.fileId = fileId; } + + @Column(name = "c_date") + public Date getDateCreated() { + return dateCreated; + } + + public void setDateCreated(Date dateCreated) { + this.dateCreated = dateCreated; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java index 4be686b2c..3a03aa8d1 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java @@ -1401,6 +1401,7 @@ public class WeiXinEvaluationController { reportFileAttach.setFileClass(model.getFileClass()); reportFileAttach.setFileName(file.getOriginalFilename()); reportFileAttach.setFileId(id); + reportFileAttach.setDateCreated(new Date()); reportFileAttachService.save(reportFileAttach); fileDao.save(id,"upload","o", file.getBytes(), contentType); From 24b435685d69add65241cebad559e756ea6654ad Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Mon, 28 Nov 2022 17:25:29 +0800 Subject: [PATCH 067/112] =?UTF-8?q?=E6=96=B0=E5=A2=9Ejenkins=E6=96=87?= =?UTF-8?q?=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Jenkinsfile | 71 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 000000000..1b1725b85 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,71 @@ +#!/usr/bin/env groovy +pipeline { + agent any + parameters { + choice(name: 'PROJECT_NAME', choices: ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic'], description: '璇烽�夋嫨鍙戝竷鐨勯」鐩�') + } + tools { + jdk 'JDK8' + } + + stages { + stage("Git Checkout") { + steps { + git changelog: false, credentialsId: 'gitlabsecret', poll: false, url: 'http://gitlab.fangjia.com/dev0/main.git' + } + } + stage('Maven Build') { + steps { + def projects = ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic'] + for (def project : projects) { + dir('deliverables/${project}') { + withMaven(globalMavenSettingsConfig: '426b9a25-aa0d-47d3-a24c-8abdfc4421ff') { + sh 'mvn clean package -Dmaven.test.skip=true' + } + } + } + } + } + stage('Docker Build') { + steps { + script { + def projects = ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic'] + def imagesName = ['fangjiabkoff': 'fangjia-bkoff', 'chafangtong': 'chafangtong', 'fangjia': 'fangjia-coresite', 'fangjiapds': 'fangjia-pds', 'fangjiastatic': 'fangjia-static'] + for (def project : projects) { + if (params.PROJECT_NAME == project) { + dir('projects/${imagesName["project"]}') { + withDockerRegistry(credentialsId: 'DockerNexus', url: 'http://192.168.0.19:9012') { + sh "docker build -t 192.168.0.19:9011/${imagesName[project]}:${env.BUILD_TAG.toLowerCase()} ." + withDockerRegistry(credentialsId: 'DockerNexus', url: 'http://192.168.0.19:9011') { + sh "docker push 192.168.0.19:9011/${imagesName[project]}:${env.BUILD_TAG.toLowerCase()}" + } + } + } + } + } + } + } + } + stage('K8S Deploy') { + steps { + script { + def projects = ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic'] + def k8s_config_map = ['fangjiabkoff': 'fangjia-bkoff', 'chafangtong': 'chafangtong', 'fangjia': 'fangjia-coresite', 'fangjiapds': 'fangjia-pds', 'fangjiastatic': 'fangjia-static'] + for (def project : projects) { + if (params.PROJECT_NAME == project) { + dir("deliverables/k8s") { + withKubeConfig([credentialsId: 'K8SSecret', serverUrl: 'https://k8s.fangjia.com:6443']) { + def orgVersion = "${k8s_config_map[project]}:latest" + def nowVersion = "${k8s_config_map[project]}:${env.BUILD_TAG.toLowerCase()}" + sh "sed -i 's/${orgVersion}/${nowVersion}/g' ${k8s_config_map[project]}.yaml" + sh "kubectl apply -f ${k8s_config_map[project]}.yaml" + } + } + } + } + } + } + } + } + +} \ No newline at end of file From 95ade7a506c016c007adf13a21ea977daa8269e8 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Mon, 28 Nov 2022 17:34:43 +0800 Subject: [PATCH 068/112] =?UTF-8?q?=E6=96=B0=E5=A2=9Ejenkins=E6=96=87?= =?UTF-8?q?=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 1b1725b85..d849b8ef3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,7 +5,7 @@ pipeline { choice(name: 'PROJECT_NAME', choices: ['fangjiabkoff', 'chafangtong', 'fangjia', 'fangjiapds', 'fangjiastatic'], description: '璇烽�夋嫨鍙戝竷鐨勯」鐩�') } tools { - jdk 'JDK8' + jdk 'JDK6' } stages { From e2748b495b1f547e5487fcfc637a060d4b262cf0 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 29 Nov 2022 10:06:43 +0800 Subject: [PATCH 069/112] =?UTF-8?q?=E8=B4=A2=E5=8A=9B=E8=AF=84=E7=BA=A7?= =?UTF-8?q?=E9=A1=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controllers/SolutionController.java | 7 + .../src/main/webapp/header/v5/_header.ftl | 2 + .../src/main/webapp/solution/financial.ftl | 200 ++++++++++++++++++ .../enums/NewResourceRecommendEnum.java | 3 +- .../mongo/fangjia/enums/SalesLeadEnum.java | 3 +- .../webapp/images/fangjia/v5/f-level-1.png | Bin 0 -> 35226 bytes .../webapp/images/fangjia/v5/f-level-2.png | Bin 0 -> 287711 bytes .../webapp/images/fangjia/v5/f-level-3.png | Bin 0 -> 104777 bytes .../webapp/images/fangjia/v5/f-level-4.png | Bin 0 -> 44035 bytes .../webapp/images/fangjia/v5/f-level-5.png | Bin 0 -> 57980 bytes .../images/fangjia/v5/f-level_banner.png | Bin 0 -> 2704791 bytes 11 files changed, 213 insertions(+), 2 deletions(-) create mode 100644 projects/fangjia-coresite/src/main/webapp/solution/financial.ftl create mode 100644 projects/fangjia-static/src/main/webapp/images/fangjia/v5/f-level-1.png create mode 100644 projects/fangjia-static/src/main/webapp/images/fangjia/v5/f-level-2.png create mode 100644 projects/fangjia-static/src/main/webapp/images/fangjia/v5/f-level-3.png create mode 100644 projects/fangjia-static/src/main/webapp/images/fangjia/v5/f-level-4.png create mode 100644 projects/fangjia-static/src/main/webapp/images/fangjia/v5/f-level-5.png create mode 100644 projects/fangjia-static/src/main/webapp/images/fangjia/v5/f-level_banner.png diff --git a/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/controllers/SolutionController.java b/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/controllers/SolutionController.java index ccc5285c3..7479e3e02 100644 --- a/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/controllers/SolutionController.java +++ b/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/controllers/SolutionController.java @@ -107,4 +107,11 @@ public class SolutionController { */ public Object paimaiAction(FangjiaModel model){return model;} + /** + * 财力评级解决方案 + * @param model + * @return + */ + public Object financialAction(FangjiaModel model){return model;} + } diff --git a/projects/fangjia-coresite/src/main/webapp/header/v5/_header.ftl b/projects/fangjia-coresite/src/main/webapp/header/v5/_header.ftl index f84257629..6b362fba0 100644 --- a/projects/fangjia-coresite/src/main/webapp/header/v5/_header.ftl +++ b/projects/fangjia-coresite/src/main/webapp/header/v5/_header.ftl @@ -32,6 +32,7 @@
  • 琛屼笟浼佷笟
  • 鍜ㄨ鐮旂┒
  • 鍙告硶鎷嶅崠
  • +
  • 璐㈠姏璇勭骇