From 1ccce2f38ae0ba8188a63686d21d5842ce8691c2 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 17 Feb 2023 14:06:44 +0800 Subject: [PATCH] update --- .../BigdataCommbuildtypeBatch.groovy | 43 +- .../BigdataCommunityBatch.groovy | 26 +- .../ccbevaluation/BigdataDealBatch.groovy | 74 +- .../ccbevaluation/BigdataForsaleBatch.groovy | 28 +- .../ccbevaluation/CcbFtpTestUtils.groovy | 2 +- .../DealAndThirdPriceDataProcess.groovy | 8 +- .../ExportOracleAllDataTest.groovy | 5 +- .../ExportOracleEvaluationData.groovy | 3 +- .../ccbevaluation/PgsqlCommUtils.groovy | 20 + .../fangjia/script/ccbevaluation/Test.groovy | 101 ++- .../script/ccbevaluation/TradeStatLog.groovy | 12 +- .../pgsql/ExportPgsqlEvaluationResult.groovy | 39 +- .../pgsql/ExportPgsqlRonghe.groovy | 604 +++++++++++++++ .../script/chongqing/HailanTest.groovy | 703 ++++++++++++++++++ .../script/chongqing/MingShenPocStat.groovy | 328 ++++++++ .../script/chongqing/MingShenPocTest.groovy | 459 ++++++++++++ .../script/chongqing/ThreeDESUtils.groovy | 105 +++ .../fuzhou/DistrictSellStatBatch.groovy | 2 + .../com/fangjia/script/fuzhou/Test1.groovy | 2 +- .../fangjia/script/gj/AddAvgpriceBatch.groovy | 1 + .../script/gj/ShLongShangHangLocation.groovy | 52 ++ .../gj/es/FangjiaCorestieLogAnaly.groovy | 42 ++ .../gj/paimai/BeiKeCrawlByChrome.groovy | 4 +- .../gj/paimai/LianjiaCrawlByChrome.groovy | 2 +- .../script/wenyan/temp/DongGuanBank.groovy | 6 +- 25 files changed, 2561 insertions(+), 110 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocStat.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/ThreeDESUtils.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ShLongShangHangLocation.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/FangjiaCorestieLogAnaly.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy index 066318e29..90803d4ca 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy @@ -7,6 +7,7 @@ import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.ccb.CcbCodeUtils import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.util.DateUtils +import groovy.sql.Sql import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService @@ -22,6 +23,9 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified", + "postgres", "postgres", "org.postgresql.Driver") + static main(args) { runBatch(new BigdataCommbuildtypeBatch(), args); } @@ -29,14 +33,14 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ @Override public void process(Object args) { def citys = CcbCommonUtils.getRunningCitys(args) - citys = citys.unique() def rets = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() for(def city : citys){ // def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) // def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","build_type", "lng","c_date", "m_date"]) - def distIds = CcbCommonUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city)); +// def distIds = CcbCommonUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city)); + def distIds = PgsqlCommUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city)) def rows = createDatas(cityCodeDatas.get(city), distIds) rets.addAll(rows) println city +" "+rows.size() @@ -50,7 +54,9 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ def headers = []//["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"] FileUtils.batchCreateFile(rets, fileName,headers, "utf-8", false); - println "txt 生成" + println "txt 生成->导Oracle" + + saveToPgSql(rets) } @@ -103,4 +109,35 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ } return rets; } + + + def saveToPgSql(rows){ + def sqls = [] + for(def r : rows) { + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ + r[i] = null + } + } + r << null + r << '2023-02-28' + r << null + r << '2999-12-31' + r << null + r << null + r << null + } + + def sql = "insert into base.t0831_bigdata_commbuildtype_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + println "start: "+new Date() + def index =0 + sqlInstance.withBatch(20000, sql){ ps-> + for(def r : rows) { + index++ + println index + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14]) + } + } + println "end: "+new Date() + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 1580dda9b..123ada0cf 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -41,7 +41,7 @@ class BigdataCommunityBatch extends BaseBatch{ @Override public void process(Object args) { def citys = CcbCommonUtils.getRunningCitys(args) -// citys = ["文昌"] + citys = ["上海"] def rows = [] def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途", @@ -247,8 +247,7 @@ class BigdataCommunityBatch extends BaseBatch{ println "start: "+new Date() def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" -// def sql = "insert into base.t0831_bigdata_community_gj_h values " - def index = 0 + sqlInstance.withBatch(20000, sql){ ps-> for(def r : rows) { ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], @@ -256,26 +255,7 @@ class BigdataCommunityBatch extends BaseBatch{ ps.toString() } } - for(def r : rows){ - index ++ - if(index == 1){ - sql = sql + "("+r[0].toString()+","+r[1].toString()+","+r[2].toString()+")" - }else{ - sql = sql + ",("+")" - } - } -// sqlInstance.withBatch(50000, sql){ ps-> -// def index = 0 -// for(def r : rows){ -// index++ -// for(def i =0 ;i + count = it.count + }); + return count; + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy index 81958589e..00ed6bf6e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy @@ -7,7 +7,7 @@ import org.apache.commons.codec.digest.DigestUtils class CcbFtpTestUtils { - static def ftpServer = "192.168.0.86"//45 + static def ftpServer = "192.168.0.45"//45 static def ftpPort = 17495 static def ftpUser = "ccb" static def ftpPassword = "ccb!@#" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index 6b28570f1..3123dc829 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -25,9 +25,9 @@ class DealPriceDataProcess { def task = new DealPriceDataProcess() // task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据 // task.addLianJiaPrice(); -// task.deleteThirdPartyData() + task.deleteThirdPartyData() // task.delDealByPe() - task.delDealData() +// task.delDealData() } def delDealByPe(){ @@ -372,12 +372,12 @@ class DealPriceDataProcess { def deleteThirdPartyData(){ def citys = CcbCommonUtils.citys_all - def month = 202211 + def month = 202212 def rows = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def set = new HashSet() for(def city : citys) { - def rets = findPeGteThirdParty(city, 202212) + def rets = findPeGteThirdParty(city, 202301) def total = 0 println city +" "+rets.size() if(rets){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy index 39ef500ac..4df84ea86 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy @@ -13,6 +13,7 @@ import org.apache.commons.codec.digest.DigestUtils import java.sql.ResultSet +//估值 新版 导出所有数据 class ExportOracleAllDataTest { def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", @@ -20,7 +21,7 @@ class ExportOracleAllDataTest { def cityCodeMap = CcbCommonUtils.findCityCodeValue() def citys = null - def curDate = "20221228"//DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def curDate = "20220128"//DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; static main(args) { def task = new ExportOracleAllDataTest() @@ -38,7 +39,7 @@ class ExportOracleAllDataTest { def run(args) { citys = CcbCommonUtils.getRunningCitys(args)//["潍坊","济宁","威海","临沂","绵阳","陇南","塔城","阿勒泰","迪庆州","日喀则"] //CcbCommonUtils.getRunningCitys(args) - def curMonth = "202212"//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); + def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") def new_dists = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy index 419f170b0..513e7025c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy @@ -31,7 +31,7 @@ class ExportOracleEvaluationData { exportCityPrice(curMonth) exportRegionPrice(curMonth) - exportCommunityBasePriceGj(curMonth)//已合下面的合并 + // exportCommunityBasePriceGj(curMonth)//已合下面的合并 exportCommunityBasePrice(curMonth) exportCommunityPdescribeGj(curMonth) @@ -248,6 +248,7 @@ class ExportOracleEvaluationData { def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy new file mode 100644 index 000000000..f2a564cb7 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy @@ -0,0 +1,20 @@ +package com.fangjia.script.ccbevaluation + +import groovy.sql.Sql + +class PgsqlCommUtils { + + public static def database = "warehouse" + + public static def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/"+database+"?stringtype=unspecified", + "postgres", "postgres", "org.postgresql.Driver") + + public static def findAllDistrictIdsByOracle(def cityid){ + def sql = "select gujia_comm_id from base.t0831_bigdata_community_gj_h where urbn_id='${cityid}' and gujia_bds_bds_st=0"; + def set = new HashSet() + sqlInstance.eachRow(sql,{r-> + set << r.gujia_comm_id + }); + return set + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 14a238118..8aca6bac6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -45,8 +45,91 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl) +def citys = ["山南","黄南州","甘南","克孜勒苏","嘉峪关","海北州","阿拉善盟","怒江州","和田","日喀则","昌都","海南州","玉树州","吐鲁番","阿勒泰","三沙","塔城","甘孜州","果洛州","林芝","那曲","阿坝州","神农架"] +def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +ExcelUtils.multiThreadingModify("D:\\数据\\小区\\小区地址名单_估家科技V2 (3).xlsx",{d-> + if(d['J']){ + def cityId = cityCodeDataMap.get(d['H']) + def map = findCcbPriceCredit(sqlInstance236, cityId, d['J']) + println d['H'] +" "+d['J']+ map.toString() + if(map){ + d['Q'] = map.credit + d['R'] = map.month + d['U'] = map.price + } +// def thirdMap = findThirdPriceBySource(sqlInstance236, cityId, d['J']) +// if(thirdMap){ +// d['S'] = thirdMap.price +// d['T'] = thirdMap.month +// } + } + +}) + +def findCcbPriceCredit(sqlInstance, cityid, commid){ + def sql = "select * from (select CONFIDENCE,year_month,avg_price from community_base_price_gj where cityid='${cityid}' and commid='${commid}' order by year_month desc) WHERE ROWNUM <= 1" + def map = [:] + sqlInstance.eachRow(sql,{r-> + map.month = r.year_month + map.credit = r.CONFIDENCE + map.price = r.avg_price + }); + return map +} + +def findThirdPriceBySource(sqlInstance, cityid, commid){ + def sql = "select * from (select data_source, price_date, community_price,commid from bigdata_third_party_price_gj where cityid='${cityid}' and commid='${commid}' and data_source in ('贝壳','链家') \n" + + "order by price_date desc) WHERE ROWNUM <= 1" + def map = [:] + sqlInstance.eachRow(sql,{r-> + map.month = r.price_date + map.price = r.community_price + }); + return map +} + + + + + +for(def c : citys){ + def cityCode = cityCodeDataMap.get(c) + println c +" "+cityCode + def priceDists = findAllPriceDist(sqlInstance236,cityCode, 202301) +// + checkNoPriceDist(sqlInstance236, priceDists, cityCode) +} + +def checkNoPriceDist(sqlInstance236,priceDists, cityid){ + def commids = [] + def sql = "select commid from bigdata_community_gj where cityid='${cityid}' and status=0"; + sqlInstance236.eachRow(sql, { r -> + commids << r.commid + }) + + for(def commid : commids){ + if(!priceDists.contains(commid)){ + updateDistStatus(sqlInstance236, commid) + } + } + return commids +} + + +def updateDistStatus(sqlInstance236, commid){ + def sql = "update bigdata_community_gj set status=1 where commid='${commid}'"; + sqlInstance236.executeUpdate(sql) +} +def findAllPriceDist(sqlInstance236, cityid, yearMonth){ + def commids = [] + def sql = "select commid from community_base_price_gj where cityid='${cityid}' and year_month=${yearMonth} "; + sqlInstance236.eachRow(sql, { r -> + commids << r.commid + }) + return commids +} //def citys = ['苏州','杭州','宁波','温州','嘉兴','湖州','绍兴','金华','衢州','舟山','台州','丽水'] // //def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() @@ -1087,15 +1170,15 @@ def findThirdCommCount(cityid, sqlInstance){ //} -def fileName = "D:\\数据\\建行\\房产估值\\指导价\\国信达价格-杭州.xlsx" -ExcelUtils.multiThreadingModify(fileName, { d -> - def dist = CommonDatautils.findDistrictById(d['A']) - if(dist){ - d['C'] = dist.city - d['D'] = dist.region - d['E'] = dist.name - } -}) +//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\国信达价格-杭州.xlsx" +//ExcelUtils.multiThreadingModify(fileName, { d -> +// def dist = CommonDatautils.findDistrictById(d['A']) +// if(dist){ +// d['C'] = dist.city +// d['D'] = dist.region +// d['E'] = dist.name +// } +//}) //def fileName = "D:\\数据\\建行\\房产估值\\指导价\\0922入库数据.xlsx" //def datas = ExcelUtils.read(fileName, 0, 1) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index e87103a90..92fb500fc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -40,7 +40,7 @@ class TradeStatLog { static main(args) { def task = new TradeStatLog() -// task.run()//统计最近3个月抓取的成交数据量 + task.run()//统计最近3个月抓取的成交数据量 // task.exportTrade()//导出未入库成交 // task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) @@ -92,13 +92,13 @@ class TradeStatLog { def run(){ def citys = CcbCommonUtils.citys_all; def rets = [] - rets << ["城市","9月总量","10月总量","11月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] + rets << ["城市","10月总量","11月总量","12月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"] //def startDate = "2021-05-01" - def startDate = "2022-09-01" - def date = "2022-10-01" - def endDate = "2022-11-01" - def newDate = "2022-12-01" + def startDate = "2022-10-01" + def date = "2022-11-01" + def endDate = "2022-12-01" + def newDate = "2023-01-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy index 1508d0b9f..24254df78 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy @@ -1,8 +1,10 @@ package com.fangjia.script.ccbevaluation.pgsql import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.ccbevaluation.CcbFtpTestUtils import com.fangjia.script.ccbevaluation.CcbFtpUtils import com.fangjia.script.ccbevaluation.ronghe.SSHHelper +import com.fangjia.script.ccbevaluation.ronghe.SSHHelperTest import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.util.DateUtils @@ -43,13 +45,11 @@ class ExportPgsqlEvaluationResult { // exportCommunityPdescribeGj(curMonth) // exportCommCaseDetailGj(curMonth) // - def curDate = curMonth.toString()+"28" -// 生成MD5文件 - exportMd5() - SSHHelper jschCommand = new SSHHelper(); - System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}")); -// ok文件 - exportOkFile() + exportInOk() + + SSHHelperTest jschCommand = new SSHHelperTest(); + System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}")); + } def exportAvmEvaluationGJNB(yearMonth){ @@ -363,20 +363,31 @@ class ExportPgsqlEvaluationResult { } - def exportOkFile(){ - def dataCodeFileName = linux_prefix + "${curDate}.ok" - if(CommonDatautils.isWindow()){ - dataCodeFileName = win_prefix + "${curDate}.ok" + def exportInOk(){ + Vector files = CcbFtpTestUtils.listFileNames(null, curDate) + Iterator fileVec = files.iterator() + def fileRows = [] + while(fileVec.hasNext()){ + ChannelSftp.LsEntry file = fileVec.next() + + if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){ + fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(null, file.getFilename(), curDate) + } } - createFileToFtp("ok txt", [], dataCodeFileName, curDate, false) + def fileName = linux_prefix + "/${curDate}.ok" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "${curDate}.ok" + } + + createFileToFtp("ok txt",fileRows, fileName, curDate, false) } def createFileToFtp(tableName,allRows, fileName, curDate, isAppend){ FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", isAppend); println tableName+" 数据文件输出完成......" - CcbFtpUtils.ftpUpload(null, fileName, curDate) -// println tableName+" 上传Ftp完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println tableName+" 上传Ftp完成......" } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlRonghe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlRonghe.groovy index 8d3d01341..5a4068b7c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlRonghe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlRonghe.groovy @@ -1,14 +1,618 @@ package com.fangjia.script.ccbevaluation.pgsql +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.ccbevaluation.CcbFtpTestUtils +import com.fangjia.script.ccbevaluation.CcbFtpUtils +import com.fangjia.script.ccbevaluation.ronghe.SSHHelperTest +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.jcraft.jsch.ChannelSftp +import groovy.sql.Sql + class ExportPgsqlRonghe { + def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/postgres", + "postgres", "postgres", "org.postgresql.Driver") + + def citys = [] + + def database = "base." + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + + def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") + def linux_prefix = "/ccb_ronghe/${curDate}/" + def win_prefix = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\" + static main(args) { def task = new ExportPgsqlRonghe() task.run(args) } def run(args) { + def new_dists = [] + citys = CcbCommonUtils.getRunningCitys(args) + //按天导(每天一个文件夹) + exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date() + 1, -1), "yyyy-MM-dd"), new_dists) + if (curDate.toString().endsWith("10")) { + + exportEvaluateRate() + exportBigdataBuildprojFjw() + exportBigdataCommlayoutpicFjw() + + + exportBigdataCommunityGJRH()// + exportBigdataBuildingFJW()// + exportBigdataCommfeeGj()// + + exportBigdataCommbuildstruGJ()// + exportBigdataCommhousetypeGJ()// + exportBigdataCommbuildyearGj()// + exportBigdataCommhouseuseGJ()// + exportBigdataPositionGJ()// + exportBigdataDistrictFJW()// + exportBigdataAliasFJW()// + + + //本12.10上传10,11月所有数据,23年1月需改为所有城市按日期同步一期数据 + def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() +// exportBigdataCityPrice(queryDate) +// exportBigdataDistPrice(queryDate) +// exportBigdataCommPrice(queryDate) + + exportInOk() + +// } + +// SSHHelperTest jschCommand = new SSHHelperTest(); +// System.out.println(jschCommand.executeCommand("sh /home/ccb_ronghe/output/rh_package.sh ${curDate}")); + } + } + + + def exportBigdataDealFJW2(curDate, dealdate, new_dists){ + + def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd") + def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd") + + def fileName = "FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + } + + def allRows = [] + + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + if(curDate.toString().endsWith("10")){ + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " + + "from "+database+"t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>=to_date('${dealDate}','yyyy-MM-dd') and gujia_admndiv_id not in ${newstr}" + def rets1 = dealData(sql1) + + def sql2 = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " + + "from "+database+"t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr}" + def rets2 = dealData(sql2) + allRows.addAll(rets1) + allRows.addAll(rets2) + }else{ + def sql = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " + + "from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>=to_date('${dealDate}','yyyy-MM-dd')"; + allRows.addAll(dealData(sql)) + } + } + } + createFileToFtp("BIGDATA_DEAL_FJW", allRows, fileName, curDate, false) + } + + + + def dealData(def sql){ + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.urbn_id + data << r.urbn_id + data << r.mdl_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_str + data << r.gujia_hsno + data << r.gujia_comm_id + data << r.comm_nm + data << r.gujia_bld_id + data << r.bld_id + data << r.gujia_bld_nm + data << r.gujia_unit_id + data << r.unit_id + data << r.gujia_unit_nm + data << r.gujia_acc_id + data << r.gujia_dtl_adr + data << r.gujia_bld_yr_rc + data << r.gujia_own_val + data << r.gujia_own_yrlmt_val + data << r.gujia_prd_val + data << r.gujia_wbt_floor + data << r.gujia_lyr_tp_val + data << r.gujia_tot_floor_num + if(r.gujia_bld_tp_val){ + def arrs = r.gujia_bld_tp_val.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.gujia_bld_stc + data << r.gujiaconstructionarea + data << r.gujia_set_innr_area + data << r.gujia_acc_tp_val + data << r.gujia_room + data << r.gujia_dept + data << r.gujia_ktch_num + data << r.gujia_toliet + data << r.gujia_hs_use + data << r.gujia_indr_stc_val + data << r.gujia_twrd_val + data << r.gujia_dcrt_sttn + data << r.gujia_ases_bt_ln_num + data << r.gujia_trdtckt_prc + data << r.gujia_mdl_tot_prc + data << r.gujia_dcl_tot_prc + data << r.gujia_ases_tot_prc + data << r.gujiasigntonlinectrid + data << r.txn_dt + data << r.gujia_txn_tp_val + data << r.gujia_sell_pymtmod + data << r.gujia_mdl_cyc + data << r.gujiawthrext_lift_ind + data << r.gujiagvlndfargrdnarea + data << r.gujia_gv_lu_tbl_area + data << r.gujia_gv_sun_tbl_area + data << r.gujiagvbaywindow_area + data << r.gujiagvarea_tower_flr + data << r.gujia_gv_grg_area + data << r.gujiagvundgdroom_area + data << r.gujia_gv_othr_area + data << r.gujia_hs_src_dtsrc + data << r.intr_co_id + data << r.intr_co_nm + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + data << r.gujia_mkdam + data << r.gujia_data_tm + allRows << data + }); + return allRows + } + + def exportEvaluateRate(){ + def sql = "select gujia_ceft_id,gujia_ceft_tp,gujia_ceft,gujia_seq, gujia_urbn,rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from t0831_evaluate_rate_fjw_w"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.gujia_ceft_id + data << r.gujia_ceft_tp + data << r.gujia_ceft + data << r.gujia_seq + data << r.gujia_urbn + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + def fileName = linux_prefix + "FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" + } + + createFileToFtp("EVALUATE_RATE_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataBuildprojFjw(){ + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_BUILDPROJ_FJW", [], fileName, curDate, false) + } + + def exportBigdataCommlayoutpicFjw(){ + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMLAYOUTPIC_FJW", [], fileName, curDate, false) + } + + def exportBigdataCommunityGJRH(){ + + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" + } + def allRows = [] + for(def city : citys){ + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_comm_id,comm_nm,gujia_comm_fmnm,gujia_admndiv_id,gujia_str,gujia_hsno,gujia_dtl_adr,gujiaocpoflandtotarea,gujia_bld_tot_area,gujia_comm_tp_val,gujiawthr_new_pan_ind,gujia_grnrate,gujia_vol_rate,gujia_park_num,gujia_comm_intd,rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujiaprhsld_hspty_prc,gujiacomm_tot_rsd_num,gujiacommrntfee_avprc,gujia_comm_avprc from "+database+"t0831_bigdata_community_fjw_w where urbn_id='${cityCode}'"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_comm_id + data << r.comm_nm + data << r.gujia_comm_fmnm + data << r.gujia_admndiv_id + data << r.gujia_str + data << r.gujia_hsno + data << r.gujia_dtl_adr + data << r.gujiaocpoflandtotarea + data << r.gujia_bld_tot_area + data << r.gujia_comm_tp_val + data << r.gujiawthr_new_pan_ind + data << r.gujia_grnrate + data << r.gujia_vol_rate + data << r.gujia_park_num + data << r.gujia_comm_intd + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + createFileToFtp("BIGDATA_COMMUNITY_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataCommbuildstruGJ(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id, gujia_comm_id, gujia_bld_stc, rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+"t0831_bigdata_commbuildstru_fjw_w where urbn_id='${cityCode}'"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_comm_id + data << r.gujia_bld_stc + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMBUILDSTRU_FJW", allRows, fileName, curDate, false) } + def exportBigdataCommfeeGj(){ + + def allRows = [] + for(def city : citys){ + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_comm_id,co_id,gujia_eps_val,gujia_hs_use,gujia_bld_tp_val,gujia_comm_eps_tp,gujia_eps_cmnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_commfee_fjw_w where urbn_id='${cityCode}'"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_comm_id + data << r.co_id + data << r.gujia_eps_val + data << r.gujia_hs_use + if(r.gujia_bld_tp_val){ + def arrs = r.gujia_bld_tp_val.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.gujia_comm_eps_tp + data << r.gujia_eps_cmnt + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMFEE_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataCommhousetypeGJ(){ + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_comm_id,gujia_comm_hs_tp,rmrk_dsc,to_char(data_crt_tm'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_commhousetype_fjw_w where urbn_id='${cityCode}'"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_comm_id + data << r.gujia_comm_hs_tp + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMHOUSETYPE_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataCommbuildyearGj(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_comm_id,gujia_bld_yr_rc,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_commbuildyear_fjw_w where urbn_id='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_comm_id + data << r.gujia_bld_yr_rc + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMBUILDYEAR_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataCommhouseuseGJ(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_comm_id,gujia_hs_use,gujia_ladder_accnum,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_commhouseuse_fjw_w where urbn_id='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_comm_id + data << r.gujia_hs_use + data << r.gujia_ladder_accnum + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMHOUSEUSE_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataPositionGJ(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_obj_tp,gujia_obj_id,gujia_lo_tp,gujia_sn,lgt,ltt,gujia_hignas_val,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_position_fjw_w where urbn_id='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_obj_tp + data << r.gujia_obj_id + data << r.gujia_lo_tp + data << r.gujia_sn + data << r.lgt + data << r.ltt + data << r.gujia_hignas_val + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_POSITION_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataDistrictFJW(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_admndiv_id,gujia_admndiv_nm,gujia_admndiv_nm_cpa,urbn_id,gujia_natstd_cd,gujia_dspl_seq,gujia_admndiv_cgy,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_district_fjw_w where urbn_id='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_admndiv_nm_cpa + data << r.urbn_id + data << r.gujia_natstd_cd + data << r.gujia_dspl_seq + data << r.gujia_admndiv_cgy + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_DISTRICT_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataAliasFJW(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,rel_id,gujia_comm_id,comm_nm,gujia_pcsst,gujia_vld_dt,gujia_vrsn_dsc,gujia_wthr_prim_nm,gujia_dtsrc,gujia_dsc,gujia_als_lvl,gujia_adr_tp_ecd,gujia_wthr_adt_ind,gujia_wthr_dspl_ind from "+database+"t0831_address_alias_fjw_w where urbn_id='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.rel_id + data << r.gujia_comm_id + data << r.comm_nm + data << r.gujia_pcsst + data << r.gujia_vld_dt + data << r.gujia_vrsn_dsc + data << r.gujia_wthr_prim_nm + data << r.gujia_dtsrc + data << r.gujia_dsc + data << r.gujia_als_lvl + data << r.gujia_adr_tp_ecd + data << r.gujia_wthr_adt_ind + data << r.gujia_wthr_dspl_ind + allRows << data + }); + } + def fileName = linux_prefix + "FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("ADDRESS_ALIAS_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataBuildingFJW(){ + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" + } + def index = 0 + for(def city : citys) { + def allRows = [] + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_bld_id,bld_id,gujia_bld_nm,gujia_bld_als_dsc,gujia_admndiv_id,gujia_comm_id,gujia_bld_yr_rc,gujia_bld_tp_val,gujia_bld_stc,floor_num,gujia_unit_num,gujia_accnum_val,gujiawthrext_lift_ind,gujia_lift_num,gujiajiladderjiactval,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_vld_ind from "+database+"t0831_bigdata_building_fjw_w where urbn_id='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_bld_id + data << r.bld_id + data << r.gujia_bld_nm + data << r.gujia_bld_als_dsc + data << r.gujia_admndiv_id + data << r.gujia_comm_id + data << r.gujia_bld_yr_rc + if(r.gujia_bld_tp_val){ + def arrs = r.gujia_bld_tp_val.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.gujia_bld_stc + data << r.floor_num + data << r.gujia_unit_num + data << r.gujia_accnum_val + data << r.gujiawthrext_lift_ind + data << r.gujia_lift_num + data << r.gujiajiladderjiactval + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + data << r.gujia_vld_ind + allRows << data + }); + + if(allRows && allRows.size()>0){ + index = index + 1 + if(index == 1){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + } + } + } + + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_BUILDING_FJW 上传Ftp完成......" + } + + def exportBigdataCityPrice(queryDate){ + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat" + } + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + // def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";// + def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from t0831_city_price_gj_h where CITYID='${cityCode}'";// + + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.YMID + data << r.SALEPRICE?r.SALEPRICE:'' + data << r.RENTPRICE?r.RENTPRICE:'' + if(r.SALECOUNT>0) { + data << r.SALECOUNT + }else{ + data << '' + } + if(r.RENTCOUNT>0){ + data << r.RENTCOUNT + }else{ + data << '' + } + data << r.UPDATETIME + data << r.HOUSEUSE + data << r.SALEBLDGAREA?r.SALEBLDGAREA:'' + data << r.RENTBLDGAREA?r.RENTBLDGAREA:'' + data << r.SALETOTALPRICE?r.SALETOTALPRICE:'' + data << r.RENTTOTALPRICE?r.RENTTOTALPRICE:'' + data << r.DATATYPE + data << r.SDATERANGE + data << r.RDATERANGE + data << r.SOURCECODE + allRows << data + }); + } + + createFileToFtp("BIGDATA_CITYPRICE", allRows, fileName, curDate, false) + } + + def exportInOk(){ + Vector files = CcbFtpTestUtils.listFileNames("rh", curDate) + Iterator fileVec = files.iterator() + def fileRows = [] + while(fileVec.hasNext()){ + ChannelSftp.LsEntry file = fileVec.next() + + if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){ + fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength("rh", file.getFilename(), curDate) + } + } + def fileName = linux_prefix+"${curDate}.ok" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "${curDate}.ok" + } + createFileToFtp("融合统计文件长度ok文件", fileRows, fileName, curDate, false) + } + + def createFileToFtp(tableName,allRows, fileName, curDate, isAppend){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", isAppend); + println tableName+" 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload("rh", fileName, curDate) + println tableName+" 上传Ftp完成......" + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy new file mode 100644 index 000000000..2b163d322 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy @@ -0,0 +1,703 @@ +package com.fangjia.script.chongqing + +import com.fangjia.core.RegionData +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import net.sf.json.JSONObject +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Matcher +import java.util.regex.Pattern + +class HailanTest { + + static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"] + private static final HashMap> FITMENTS_TYPES_MAPS = new HashMap>() + static{ + FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"]) + FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"]) + FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"]) + FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"]) + } + + static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) + def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + + static main(args) { + def task = new HailanTest(); + task.run(); + } + + def run(){ + def inputfile = "D:\\数据\\海南\\统计.xlsx" +// match(inputfile) +// addPrice(inputfile) +// addCommId("D:\\数据\\海南\\匹配上的合并数据.xlsx") +// addHouse(inputfile) +// stat(inputfile) + hailanStat(inputfile) + } + + def addHouse(inputfile){ + + def datas = [] + datas << ["类型(住宅/别墅)","行政区划","详细地址","房号","小区名称","建筑面积","所在楼层","详细地址","小区名","交易日居住用房价格指数","评估日居住用房价格指数","是否有电梯","所在楼层","总楼层","内部装修","平面布置","朝向", + "景观","建筑结构","建筑面积(平方米)","房屋可使用年限","房屋已使用年限","评估价(单价)","挂牌价(单价)","成交价(单价)","小区/楼盘均价(单价)"] + def index = 0 + ExcelUtils.multiThreadingModify(inputfile, { row -> + if(row['L']){ + index ++ + def dist = CommonDatautils.findDistrict(row['H'], row['I'], row['J'], "district") + def rets = findHouses(row['H'], row['I'], "sell", 20200101,20230113, row['J'], null, row['k']) + println index +" "+ row['C']+" "+ rets.size() + if(rets){ + def lastRets = [] + for(def r : rets){ + def fjwPrice = row['M'] + if(fjwPrice){ + def precent = java.lang.Math.abs((fjwPrice-r.avg_price)*100/r.avg_price) + if(precent>=10){ + continue; + } + } + if(row['A'] && row['A']=="别墅"){ + def inputAare = row['F'].toString().toDouble() + def minArea = inputAare + 50 + def maxArea = inputAare + 100 + if(r.areamaxArea){ + continue; + } + } + + lastRets << r + if(lastRets.size()>=3){ + break; + } + } + + if(lastRets){ + for(def lr : lastRets){ + def data = [] + data << row['A'] + data << row['B'] + data << row['C'] + data << row['D'] + data << row['E'] + data << row['F'] + data << row['G'] + if(dist){ + data << dist.address + }else{ + data << "" + } + if(dist){ + data << dist.name + }else{ + data << "" + } + + data << "" + data << "" + if(lr.height){ + if(lr.height>6){ + data << "是" + }else{ + data << "否" + } + }else{ + data << "" + } + data << lr.floor + data << lr.height + data << lr.fitment + data << "" + data << lr.direction + data << "" + data << lr.build_type + data << lr.area + data << dist.property_life +// def year = dist.create_year +// if(year){ +// data << getHouseYear(year.toInteger())//已使用年限 +// }else{ +// data << ""//已使用年限 +// } + data << getHouseYear(dist.create_year , lr.build_date, dist.build_date) + + data << getEvalutaionPrice(lr, dist)//评估价(单价) + data << lr.avg_price + data << getDealPrice(dist)//成交单价 + data << row['M'] + datas << data + } + } + } + } + }) + ExcelUtils.write("D:\\数据\\海南\\海南样本数据.xlsx", datas, "样本数据") + } + + def getDealPrice(dist){ + def json = OpenApiClient.searchDealPrice(dist._id.toString(),"20220101", 100) + def prices = [] + if(json && json.code==200){ + def result = json.result + if(result){ + for(def r : result){ + prices << r.avgPrice + } + } + } + def dealAvgPrice = null + if(prices){ + dealAvgPrice = (prices.sum()/prices.size()).toInteger() + } + return dealAvgPrice; + } + + def getHouseYear(create_year , house_build_date, dist_build_date){ + def year = null + if(house_build_date){ + year = getYear(house_build_date) + + }else if(dist_build_date){ + year = getYear(dist_build_date) + }else if(create_year){ + year = create_year + } + if(year){ + year = year.toInteger() + def curYear = new Date().getYear()+1900 + def modYear = curYear - year + return modYear.toInteger().toString() + } + return null + + } + + + def getYear(finishBuildingDate){ + if(finishBuildingDate){ + Pattern pat3 = Pattern.compile("\\d{4}"); + def matcher3 = pat3.matcher(finishBuildingDate); + if(matcher3.find()){ + return matcher3.group() + } + } + return null + } + + + def getEvalutaionPrice(house, dist){ + def params = [:] + params.put("city", dist.city); + params.put("district", dist.region); + params.put("name", dist.name); + params.put("size", String.valueOf(house.area)); + if(house.height) { + params.put("maxFloor", dist.house_height.toString()); + params.put("direction", getDirectionCode(house.direction).toString());//朝向 + if (house.house_floor) { + params.put("floor", house.house_floor.toString()); + } + } + def fitmentCode = null + if(house.fitment){ + fitmentCode = getFitmentCode(house.fitment) + params.put("decoration", fitmentCode.toString());//装修 + } + params.put("scope", "1") + def result = OpenApiClient.evaluateDistrictPrice(params); + + if(result){ + return result.avgPrice + } + return null + } + + + def getDirectionCode(def s){ + def directMap = ["西南":1, "东南":2, "南":3, "西北":4, "东北":5, "东":6, "西":7, "北":8, "东西":9, "南北":10] + return directMap.get(s); + } + def getFitmentCode(def s){ + if(s){ + if(["毛坯"].contains(s)){ + return 1; + }else if(["简单装修","简装"].contains(s)){ + return 2; + }else if(["精装修","精装"].contains(s)){ + return 3; + }else if(["豪华装修","豪装"].contains(s)){ + return 4; + } + } + return null + } + + + + + def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range) + + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + + + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + + def rets = [] + def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"] + def index = new Random().nextInt(5); + + def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder); + + if(houses){ + for (int k=0;k60){ + continue; + } + +// if(!seawater.room){ +// continue; +// } +// if(!seawater.room){ +// seawater.room = null +// } + + if(seawater.room){ + if(seawater.room>=11 || seawater.room<1){ + continue; + } + } + + if(!seawater.hall){ + seawater.hall = null + } + if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){ + continue; + } + if(!seawater.toilet){ + seawater.toilet = null + } + if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){ + continue; + } + + + def floor = parseFloor(seawater.floor_info) + if(seawater.height && floor && floor.isNumber()){ + if(floor.toInteger()>seawater.height){ + floor = null + seawater.height = null + } + + } + + if(!floor){ + seawater.floor = null + } + if(floor && floor.toInteger()<1){ + floor = null + } + + def rowMap = [:]; + + rowMap.houseId = seawater.id +// rowMap.city = CcbCommonUtils.getCcbCity(city) + //rowMap.region =seawater.region + //rowMap.address = seawater.address + rowMap.area = seawater.area + rowMap.address = seawater.address + rowMap.total_price = seawater.total_price + rowMap.avg_price = seawater.avg_price + rowMap.build_type = seawater.build_type + rowMap.build_date = seawater.build_date + rowMap.district_name = seawater.district_name + rowMap.height = seawater.height=="null"?null:seawater.height + if(rowMap.height && rowMap.height<1){ + rowMap.height = null + } + rowMap.floor = floor=="null"?null:floor + rowMap.room = seawater.room=="null"?null:seawater.room + rowMap.hall = seawater.hall=="null"?null:seawater.hall + rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet + + + def direction = parseDirection(seawater.direction) + rowMap.direction = direction=="null"?null:direction + + def fitment = parseFitment(seawater.fitment) + rowMap.fitment = fitment=="null"?null:fitment + + + + if(seawater.p_date instanceof String){ + seawater.p_date = seawater.p_date.toInteger() + } + + rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") + rowMap.c_date = DateUtils.getConverDate(seawater.m_date) + rowMap.m_date = DateUtils.getConverDate(seawater.m_date) + + rets << rowMap + } + } + return rets; + } + + + static def parseFloor(def floorDesc){ + if(floorDesc){ + floorDesc = floorDesc.replaceAll("[\\s ]+", "") + if(floorDesc){ + Pattern pat1 = Pattern.compile("第(\\d{1,2})层"); + Matcher matcher1 = pat1.matcher(floorDesc); + if (matcher1.find()) { + return matcher1.group(1) + } + pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?"); + matcher1 = pat1.matcher(floorDesc); + if (matcher1.find()) { + return matcher1.group(1) + } +// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){ +// return "中层" +// } +// +// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){ +// return "高层" +// } +// +// +// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){ +// return "低层" +// } + if(floorDesc.isNumber()){ + return floorDesc; + } + } + } + return null; + } + + static String parseDirection(String directionDesc) { + if (directionDesc == null) { + return null + } + directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#") + StringBuffer sb = new StringBuffer() + for (int i = 0; i < directionDesc.length(); i++) { + String c = directionDesc.substring(i, i + 1) + if (DIRECTIONS.contains(c)) { + if (directionDesc.substring(i).startsWith(c + "#_#")) { + sb.append(c + " ") + } else { + sb.append(c) + } + } + } + for (String d : DIRECTIONS) { + if (sb.toString().startsWith(d)) { + return d + } + } + for (String d : DIRECTIONS) { + if (sb.toString().contains(d)) { + return d + } + } + return null + + } + + static String parseFitment(String fitmentDesc) { + if (fitmentDesc == null) { + return null + } + fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "") + if (fitmentDesc.equals("非毛坯")) { + return null + } + Iterator>> it = FITMENTS_TYPES_MAPS + .entrySet().iterator() + Map.Entry> entry + while (it.hasNext()) { + entry = it.next() + if (fitmentDesc.contains(entry.getKey())) { + return entry.getKey() + } + for (String fitmentKey : entry.getValue()) { + if (fitmentDesc.contains(fitmentKey)) { + return entry.getKey() + } + } + } + return null + + } + + + + def addCommId(inputfile){ + ExcelUtils.modify(inputfile, { row -> + if(row['H']){ + def dist = CommonDatautils.findDistrict(row['H'], row['I'], row['J'], "district") + if(dist){ + row['L'] = dist._id.toString() + } + } + }) + } + + def addPrice(path){ + ExcelUtils.multiThreadingModify(path, { row -> + def commid = row['L'] + if(commid){ + def dist = CommonDatautils.findDistrictById(commid.trim()) + if(dist){ + def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name) + if(priceObj){ + row['M'] = priceObj.avg_price + } + } + } + }) + } + + + def match(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(row['C']){ + String city = row['H']//RegionData.formatCity(row['B']) + if(city){ + def address = row['C'] +// def region = row['C'] + println "[match] city:"+city+" address:"+address + def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, null, row['E']) + if(jsonObject != null && jsonObject.precise!=-1) { + row['I'] = jsonObject.get("district") + row['J'] = jsonObject.get("name") + row['K'] = jsonObject.get("propertyType") + row['L'] = jsonObject.get("id") + } + }else{ + println row['C'] + } + + } + }) + } + + def stat(path){ + ExcelUtils.modify(path, { row -> + if(row['L'] && !row['H']){ + def city = row['L'] + def region = row['M'] + def dists = null + if(city && region){ + dists = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city","region","name","alias","estate_type2"]) + }else{ + dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias","estate_type2"]) + } + def distLists = [] + def villaLists = [] + for(def d : dists){ + if(d.estate_type2=="别墅"){ + villaLists << d + }else{ + distLists << d + } + } + row['J'] = dists.size() + + def haveHouseList = findHaveHouseDist(distLists) + def haveVillaList = findHaveHouseDist(villaLists) + row['H'] = haveHouseList + row['I'] = haveVillaList +// row['N'] = distLists.size() +// row['O'] = villaLists.size() + } + }) + } + + def hailanStat(path){ + def total = 0 + def distTotal = 0 + def villaTotal = 0 + def haveHouseDistTotal = 0 + def haveHouseVillaTotal = 0 + def buildStructureTotal = 0 +// ExcelUtils.modify(path, { row -> +// if (row['L'] && !row['K']) { + //"海口","三亚","三沙","儋州", + def citys = ["五指山","琼海","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"] + for(def city : citys) { + + def dists = null +// if (city && region) { +// dists = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city", "region", "name", "alias", "estate_type2", "build_structure"]) +// } else { + dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias", "estate_type2", "build_structure"]) +// } + def distLists = [] + def villaLists = [] + for (def d : dists) { + if (d.estate_type2 == "别墅") { + villaLists << d + } else { + distLists << d + } + } + total = total + dists.size() + +// def structureCount = 0 +// for(def d : dists){ +// if(d.build_structure){ +// structureCount ++ +// } +// } +// println structureCount +// row['K'] = structureCount +// buildStructureTotal = buildStructureTotal + structureCount +// + def haveHouseList = findHaveHouseDist(distLists) + def haveVillaList = findHaveHouseDist(villaLists) + haveHouseDistTotal = haveHouseDistTotal + haveHouseList + haveHouseVillaTotal = haveHouseVillaTotal + haveVillaList + distTotal = distTotal + distLists.size() + villaTotal = villaTotal + villaLists.size() +// println c +" "+total+" "+buildStructureTotal +// } + } +// }) + + println total +" "+haveHouseDistTotal+" "+haveHouseVillaTotal+" "+distTotal+" "+villaTotal+" "+buildStructureTotal + + } + + def findHaveHouseDist(districts){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + println "小区量:"+districts.size() + + def rets = new CopyOnWriteArrayList() + + for(def i=0;i0){ + rets.add(dist) + } + println totalCount.get() +" "+ ret.size() + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + }; + }) + } + latch.await() + return rets.size() + } + + + def findThirdPrice(dist){ + def q = new BasicDBObject() + q.put("city", dist.city) + q.put("region", dist.region) + q.put("name", dist.name) + q.put("s_date", new BasicDBObject('$gte', 202201)) + def dbcursor = AVERAGE_STATISTICS.find(q) + def prices = [] + if(dbcursor){ + for(def d : dbcursor){ + def set = d.keySet(); + Iterator it = set.iterator() + while(it.hasNext()){ + def key = it.next() + if(key && key.contains("_price") && key!="fangjia_price" && key!="last_price"){ + def price = d[key] + if(price){ + prices << price + } + } + } + } + } + return prices; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocStat.groovy new file mode 100644 index 000000000..d1e0db444 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocStat.groovy @@ -0,0 +1,328 @@ +package com.fangjia.script.chongqing + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.rabbitmq.client.impl.AMQImpl +import org.apache.lucene.queryparser.classic.QueryParser +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + + +class MingShenPocStat { + + static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + + private static final DBCollection district_sell_stat_coll = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com") + final static DBCollection CITY_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", MongoUtils.MongoDB_Domain_mongo); + + static main(args) { + def task = new MingShenPocStat() +// task.run() + task.officeRun() + } + + def officeRun(){ + def path = "D:\\数据\\民生\\抽样-写字楼类.xlsx" + def datas = [] + ExcelUtils.modify(path, { d -> + if(d['B']){ + + def office = CommonDatautils.findDistrictById(d['C']) + if(office){ + def alias = [] + alias << office.name + if(office.alias){ + def nameAlias = office.alias + for(def o : nameAlias){ + if(o){ + alias << o + } + } + } + def aliasStr = escapeExprSpecialWord(alias.join("','")) + def houses = findOfficeHouseCount(office.city, office.region, aliasStr) + if(houses){ + if(houses.size()>=2){ + houses = houses[0..1] + } + for(def h : houses){ + def obj = findOfficeAvgPrice(office) + def rets = [] + rets << d['A'] + rets << d['B'] + rets << '' + rets << h.region + rets << office.address + rets << h.area + rets << h.floor + rets << h.height + def level = office.bed_kind + rets << level + rets << h.fitment + def date = office.finish_building_date + if(date){ + rets << date.substring(0, 4) + }else{ + rets << "" + } + + rets << "" + rets << "" + def price = "" + if(obj && obj.avg_price){ + price = obj.avg_price + rets << obj.avg_price + }else{ + rets << "" + } + + if(price){ + rets << (price*h.area).toInteger() + } + datas << rets + } + } + }else{ + def rets = [] + rets << d['A'] + rets << d['B'] + datas << rets + } + + // def districts = CommonDatautils.findAllDistrictByCity(d['C'],"office",["city", "region", "name", "alias","address","bed_kind","finish_building_date"]) + + } + }) + + ExcelUtils.write("D:\\数据\\民生\\抽样-写字楼类-样本.xlsx", datas, "样本") + } + + def run(){ + def path = "D:\\数据\\民生\\样本字段覆盖率.xlsx" +// def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Downloads\\二期估值数据每周统计\\二期估值数据每周统计\\估值数据统计结果20230112.xlsx", 0 , 1) +// +// ExcelUtils.modify(path, { row -> +// if(row['C']) { +// def total = 0 +// def pre = 0 +// for(def d : datas){ +// if(row['C']==d['A']){ +// total = d['B'] +// pre = d['H'] +// break +// } +// } +// if(total){ +// def c = (total*pre/100).toInteger() +// row['E'] = c +// row['F'] = pre+"%" +// } +// } +// }) + + def totalcount = findHouseTotal("大连") + def count = findHouseCount("大连", "height") + + println DataHandleUtils.getDoubleRound(2, count*100/totalcount) + +// ExcelUtils.multiThreadingModify(path, { row -> +// if(row['C']) { + //def districts = CommonDatautils.findAllDistrictByCity(row['C'],"district",["city", "region", "name", "alias"]) +// def districts = CommonDatautils.findDistrictByCityAndRegions("大连","district",["旅顺口"], ["city", "region", "name", "alias"]) +// def total = districts.size() +// if(total){ +// def house_dist_count = findHouseStat(districts) +// def pre = house_dist_count*100/total +// pre = DataHandleUtils.getDoubleRound(2, pre) +// println total+" "+house_dist_count+" "+pre +// } +// def count = findHouseCount(row['C']) +// row['D'] = count +// println row['C']+" "+count +// row['H'] = findDistrictSellStat(row['C']) +// row['G'] = getNewDist(row['C']) + +// def totalcount = findHouseTotal(row['C']) + +// def total = row['Z'] +// def count = findHouseCount(row['C'], "build_date") +// row['Q'] = DataHandleUtils.getDoubleRound(2, count*100/total) +// println count +" "+row['Q'] +// def obj = getFieldPrecent(row['C']) +// if(obj){ +// def detail = obj.district_detail +// row['E'] = detail.alias.toDouble()*100 +// row['F'] = detail.address.toDouble()*100 +// row['G'] = detail.lng.toDouble()*100 +// row['H'] = detail.buildType.toDouble()*100 +// row['I'] = detail.outPics.toDouble()*100 +// row['J'] = detail.developer.toDouble()*100 +// row['Q'] = detail.buildDate.toDouble()*100 +// row['R'] = detail.buildDate.toDouble()*100 +// } +// } +// }); + } + + def getFieldPrecent(city){ + def q = new BasicDBObject() + q.put("city", city) + def rets = CITY_INFO_CONN.find(q).sort(new BasicDBObject('weekend', -1)) + if(rets){ + return rets[0] + } + return null + } + + + def getNewDist(city){ + String sql = "select count(*) as count from community where city='${city}' and source='贝壳找房-新房' " + def count = null + ElasticUtils.executeSqlQuery(sql,{ data-> + count = data.count + }) + return count; + } + + def findDistrictSellStat(def city){ + def q = new BasicDBObject() + q.put("city", city) + def count = district_sell_stat_coll.count(q) + return count + } + + + def findHouseStat(districts){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + println "小区量:"+districts.size() + + def rets = new CopyOnWriteArrayList() + + for(def i=0;i0){ + rets.add(rMap) + } + println totalCount.get() +" "+ ret.size() + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + }; + }) + } + latch.await() + return rets.size() + } + + def findHouseTotal(city){ + def count = 0 + ElasticUtils.executeSqlQuery("select count(*) as count from seawater where p_date >= 20220101 and city='${city}'", { + count = it.count + println("$city\t${it.count}") + }) + return count + } + + def findHouseCount(city, field){ + def count = 0 + def sql = "select count(*) as count from seawater where city='${city}' and cat='sell' and "+field+" is not null" + ElasticUtils.executeSqlQuery(sql, { + count = it.count + println("$city\t${it.count}") + }) + return count + } + + def findHouses(def city, def region, def cat, def startDate, def name, def alias){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(false).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range) + + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + + + if(name.contains("弄小区")){ + def newName = name.replace("弄小区","弄") + if(!alias.contains(newName)){ + alias << newName + } + + } + + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + + def rets = [] + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + return houses; + } + + def findOfficeAvgPrice(office){ + def sql = "select * from office where city='${office.city}' and region='${office.region}' and name='${office.name}'" + def rets = [] + ElasticUtils.executeSqlQuery(sql, {d-> + rets << d + }) + if(rets){ + def obj = rets[0] + return obj + } + return null + + } + + def findOfficeHouseCount(city, region, names){ + def sql = "select * from seawater where city='${city}' and region='${region}' and cat='office_sell' and district_name in ('${names}') and root='58.com'" + def rets = [] + ElasticUtils.executeSqlQuery(sql, {d-> + rets << d + }) + return rets + } + + + /** + * 转义字符串中的特殊字符 + * 仅过滤Lucene regexps, 其他的勿用 + * @param keyword + * @return + */ + public static String escapeExprSpecialWord(String keyword) { + return QueryParser.escape(keyword); + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocTest.groovy new file mode 100644 index 000000000..112de372f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocTest.groovy @@ -0,0 +1,459 @@ +package com.fangjia.script.chongqing + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.BaiduMapUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +class MingShenPocTest { + + def total = 5000 + static main(args) { + def task = new MingShenPocTest() + task.run() + } + def run(){ + def path = "D:\\数据\\民生\\poc测试.xlsx" + //9级标准化 + //structureV2(path)//针对海南poc地址数据此接口不合适使用 + //省市区拆分 +// ner(path)//城市,区域带市带区 +// match(path) +// addPrice(path) +// addLatLng(path) +// addAroundPrice(path) +// addAuthor(path) + addHouse(path) + } + + def structureV2(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(row['A']){ + def jsonObj = FangjiaClientUtils.addressStructure(null, row['A']) + if(jsonObj && jsonObj.result){ + + if(jsonObj.result.province){ + row['B'] = jsonObj.result.province + } + + if(jsonObj.result.city){ + def cityStr = null + if(jsonObj.result.city=="市辖区"){ + cityStr = jsonObj.result.province + }else{ + cityStr = jsonObj.result.city + } + row['C'] = RegionData.formatCity(cityStr) + } + + if(jsonObj.result.district){ + row['D'] = jsonObj.result.district + } + } + } + }) + } + + def ner(path){ + ExcelUtils.multiThreadingModify(path, { d -> + if(d['A'] && !d['B']) { + def json = FangjiaClientUtils.addressStand(null, d['A']) + if(json && json.entities){ + def entities = json.entities + for(def m : entities){ + if(m.type=="prov"){ + d['B'] = m.word + } + if(m.type=="city"){ + def cityStr = m.word + if(cityStr){ + d['C'] = cityStr//RegionData.formatCity(cityStr) + } + } + if(m.type=="dist") { + if(d['C']){ + d['D'] = m.word //RegionData.formatRegion(d['C'], m.word) + } + } + } + } + } + }); + } + + //可以根据测试数据添加 推荐楼盘名 + def match(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(row['C'] && !row['E']){ + def city = RegionData.formatCity(row['C']) + def region = null + if(city){ + region = RegionData.formatRegion(city, row['D']) + } + def address = row['A'] + println "[match] city:"+city+" region:"+region+" address:"+address + def jsonObject = OpenApiClient.addressMatch(city, address, null, false, region, null)//"property" + if(jsonObject != null && jsonObject.precise!=-1) { + row['E'] = jsonObject.get("city") + row['F'] = jsonObject.get("district") + row['G'] = jsonObject.get("name") + row['H'] = jsonObject.get("propertyType") + row['I'] = jsonObject.get("id") + } + } + }) + } + + def addPrice(path){ + ExcelUtils.multiThreadingModify(path, { row -> + def commid = row['I'] + if(commid && !row['J']){ + def dist = CommonDatautils.findDistrictById(commid.trim()) + if(dist){ + def house = [:] + house.area = 100 + def price = getEvalutaionPrice(house, dist) + if(price){ + row['J'] = price + } +// def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name) +// if(priceObj){ +// row['J'] = priceObj.avg_price +// } + } + } + }) + } + + def getEvalutaionPrice(house, dist){ + def params = [:] + params.put("city", dist.city); + params.put("district", dist.region); + params.put("name", dist.name); + params.put("size", String.valueOf(house.area)); + if(house.height) { + params.put("maxFloor", dist.house_height.toString()); + params.put("direction", getDirectionCode(house.direction).toString());//朝向 + if (house.house_floor) { + params.put("floor", house.house_floor.toString()); + } + } + def fitmentCode = null + if(house.fitment){ + fitmentCode = getFitmentCode(house.fitment) + params.put("decoration", fitmentCode.toString());//装修 + } + params.put("scope", "1") + def result = OpenApiClient.evaluateDistrictPrice(params); + + if(result){ + return result.avgPrice + } + return null + } + + + def getDirectionCode(def s){ + def directMap = ["西南":1, "东南":2, "南":3, "西北":4, "东北":5, "东":6, "西":7, "北":8, "东西":9, "南北":10] + return directMap.get(s); + } + def getFitmentCode(def s){ + if(s){ + if(["毛坯"].contains(s)){ + return 1; + }else if(["简单装修","简装"].contains(s)){ + return 2; + }else if(["精装修","精装"].contains(s)){ + return 3; + }else if(["豪华装修","豪装"].contains(s)){ + return 4; + } + } + return null + } + + def addLatLng(path){ + ExcelUtils.multiThreadingModify(path, { d -> + if(!d['J'] && d['H'] && (d['H']=="普通住宅" || d['H']=="商住" || d['H']=="别墅")) { + def json = BaiduMapUtils.searchByAddress(d['A']) + if(json && json.location){ + d['K'] = json.location.lng + d['L'] = json.location.lat + } + } + }) + } + + + def addAroundPrice(path){ + ExcelUtils.multiThreadingModify(path, { d -> + if(!d['J'] && d['C'] && d['K']){ + def city = RegionData.formatCity(d['C']) + def dist = new BasicDBObject() + dist.city = city + dist.lng = d['K'].toString().toDouble()*10000000000L + dist.lat = d['L'].toString().toDouble()*10000000000L + dist.estateType = "普通住宅"//d['H'] + if(dist.lat) { + List list = DbUtils.findAroundDistrictByRadius(dist, 1000) + if (list) { + list = sortByDistance(dist, list); + if (list && list.size() > 10) { + list = list[0..9] + } + if (list) { + def prices = [] + for (def s : list) { + def priceObj = CommonDatautils.findDistrictPrice(s.city, s.region, s.name) + if (priceObj) { + prices << priceObj.avg_price + } + } + if (prices) { + def arround_price = (int) (prices.sum() / prices.size()) + d['J'] = arround_price + } + } + } + } + } + }); + } + + def sortByDistance(d, list){ + for(def nb : list){ + def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0) + nb.distance = distance + } + //排序 + Collections.sort(list, new Comparator() { + @Override + int compare(Object o1, Object o2) { + if(o1.distance > o2.distance) + return 1;//第二个元素(o1)比第一个元素(o2)大,返回-1 + if(o1.distance == o2.distance) + return 0; + return -1; + } + }) + return list; + } + + + def addAuthor(def path){ + def authors = ["董彦均","曹玉彦","付贵生","沈燕芳","Catherine", "王蕾","吉珉", "张宝剑"] + def personTotal = authors.size() + def avg = Math.ceil(total/personTotal) + + def index = 0 + def count = 0; + ExcelUtils.modify(path, { row -> + if(row['A']){ + if(index < avg){ + row['M'] = authors[count] + } + index++ + if(index >= avg){ + count++ + index = 0 + } + } + }); + } + + def addHouse(path){ + def datas = [] + ExcelUtils.multiThreadingModify(path, { row -> + def commId= row['E'] + if(commId){ + def dist = CommonDatautils.findDistrictById(commId) + if(dist){ + def rets = findHouses(row['H'], row['I'], "sell", 20200101,20230113, row['J'], null, row['k']) + if(rets){ + def data = [] + for(def r : rets){ + data << r.city + data << r.region + data << r.district_name + data << r.floor + data << r.height + data << r.fitment + data << r.direction + data << r.build_type + data << r.area + data << r.avg_price + datas << data + } + } + } + } + }); + + ExcelUtils.write("D:\\数据\\民生\\民生样本数据.xlsx", datas, "样本数据") + } + + def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range) + + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + + + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + + def rets = [] + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + + if(houses){ + for (int k=0;k60){ + continue; + } + +// if(!seawater.room){ +// continue; +// } +// if(!seawater.room){ +// seawater.room = null +// } + + if(seawater.room){ + if(seawater.room>=11 || seawater.room<1){ + continue; + } + } + + if(!seawater.hall){ + seawater.hall = null + } + if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){ + continue; + } + if(!seawater.toilet){ + seawater.toilet = null + } + if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){ + continue; + } + + + def floor = parseFloor(seawater.floor_info) + if(seawater.height && floor && floor.isNumber()){ + if(floor.toInteger()>seawater.height){ + floor = null + seawater.height = null + } + + } + + if(!floor){ + seawater.floor = null + } + if(floor && floor.toInteger()<1){ + floor = null + } + + def rowMap = [:]; + + rowMap.houseId = seawater.id +// rowMap.city = CcbCommonUtils.getCcbCity(city) + //rowMap.region =seawater.region + //rowMap.address = seawater.address + rowMap.area = seawater.area + rowMap.address = seawater.address + rowMap.total_price = seawater.total_price + rowMap.avg_price = seawater.avg_price + rowMap.build_type = seawater.build_type + rowMap.build_date = seawater.build_date + rowMap.district_name = seawater.district_name + rowMap.height = seawater.height=="null"?null:seawater.height + if(rowMap.height && rowMap.height<1){ + rowMap.height = null + } + rowMap.floor = floor=="null"?null:floor + rowMap.room = seawater.room=="null"?null:seawater.room + rowMap.hall = seawater.hall=="null"?null:seawater.hall + rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet + + + def direction = parseDirection(seawater.direction) + rowMap.direction = direction=="null"?null:direction + + def fitment = parseFitment(seawater.fitment) + rowMap.fitment = fitment=="null"?null:fitment + + + + if(seawater.p_date instanceof String){ + seawater.p_date = seawater.p_date.toInteger() + } + + rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") + rowMap.c_date = DateUtils.getConverDate(seawater.m_date) + rowMap.m_date = DateUtils.getConverDate(seawater.m_date) + + rets << rowMap + } + } + return rets; + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/ThreeDESUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/ThreeDESUtils.groovy new file mode 100644 index 000000000..3d685a1a5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/ThreeDESUtils.groovy @@ -0,0 +1,105 @@ +package com.fangjia.script.chongqing + +import com.fangjia.script.util.ExcelUtils +import org.apache.commons.io.FileUtils + +import javax.crypto.Cipher; +import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; +import java.util.Base64; + +/** + * 3DES加密工具类 + * @author qzz + */ +public class ThreeDESUtils { + + /** + * 加解密统一编码方式 + */ + private final static String ENCODING = "utf-8"; + + /** + * 加解密方式 + */ + private final static String ALGORITHM = "DESede"; + + /** + *加密模式及填充方式 + */ + private final static String PATTERN = "DESede/ECB/pkcs5padding"; + + /** + * 3DES加密 + * + * @param plainText 普通文本 + * @param sK 秘钥(24位密码) + * @return + * @throws Exception + */ + public static String encode(String plainText,String sK) throws Exception { + SecretKey secretKey = new SecretKeySpec(sK.getBytes(ENCODING), ALGORITHM); + // 3DES加密采用pkcs5padding填充 + Cipher cipher = Cipher.getInstance(PATTERN); + // 用密匙初始化Cipher对象 + cipher.init(Cipher.ENCRYPT_MODE, secretKey); + // 执行加密操作 + byte[] encryptData = cipher.doFinal(plainText.getBytes(ENCODING)); + return Base64.getEncoder().encodeToString(encryptData); + } + + /** + * 3DES解密 + * + * @param encryptText 加密文本 + * @return + * @throws Exception + */ + public static String decode(String encryptText, String sK) throws Exception { + SecretKey secretKey = new SecretKeySpec(sK.getBytes(ENCODING), ALGORITHM); + // 3DES加密采用pkcs5padding填充 + Cipher cipher = Cipher.getInstance(PATTERN); + + // 用密匙初始化Cipher对象 + cipher.init(Cipher.DECRYPT_MODE, secretKey); + // 正式执行解密操作 + byte[] decryptData = cipher.doFinal(Base64.getDecoder().decode(encryptText)); + return new String(decryptData, ENCODING); + } + + public static void main(String[] args) throws Exception { + + def rets = [] + def datas = readTxt("D:\\数据\\民生\\抽样-住宅类(普通住宅、公寓).txt") + def index = 0 + for(def d : datas){ + index++ + def row = [] + def str = decode(d,"111036369260679051122113") + println str + def id = str.substring(str.length()-4) + row << id + row << str.substring(0, str.indexOf(id)) + rets << row + } + ExcelUtils.write("D:\\数据\\民生\\民生银行测试数据.xlsx", rets, "抽样-住宅类(普通住宅、公寓)") + +// ExcelUtils.multiThreadingModify("D:\\数据\\民生\\加密数据.xlsx", { row -> +// //加密 +// String str = row['A'] +// //解密 +// row['B'] = (decode(str,"111036369260679051122113")); +// }); + + } + + public static def readTxt(def file){ + def lines = FileUtils.readLines(new File(file)) + def ret = []; + for(line in lines) { + ret << line; + } + return ret; + } +} + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy index 088626285..1b604d376 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy @@ -18,6 +18,7 @@ import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Pattern //小区月度均价表(带涨跌幅控制) @BatchOwner(Watcher.GaoJuan) @@ -78,6 +79,7 @@ class DistrictSellStatBatch extends BaseBatch { }else{ districts = DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000) } + print city +" "+districts.size() def weekendPriceMap = findPriceMap(city, lastMonthWeekend)//上个月小区均价 def curMonthlyPriceMap = findMonthlyPrice(city, curMonth)//当月小区月均价 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index 7e4a0bd2c..44e4dd54c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202211.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202301.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy index 577aac71d..7a531b0e7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy @@ -51,6 +51,7 @@ class AddAvgpriceBatch { for(def disKey : res2){ def array = disKey.split("##"); def estateType2 = districtKeysMap.get(disKey) + //发重算均价命令 CommonUtils.recalculationAverage(array[0], array[1], array[2], estateType2, 0, new Date()) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ShLongShangHangLocation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ShLongShangHangLocation.groovy new file mode 100644 index 000000000..4cc9588c1 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ShLongShangHangLocation.groovy @@ -0,0 +1,52 @@ +package com.fangjia.script.gj + +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils + +class ShLongShangHangLocation { + static main(args) { + def task = new ShLongShangHangLocation() + task.run() + } + + def run(){ + def rets = ["上海市黄兴路1599号","上海市杨浦区长阳路1687号A栋","上海市杨浦区佳木思路432号","上海市杨浦区控江路1063号","上海市杨浦区鞍山路100号","上海市杨浦区延吉东路287号","上海市杨浦区殷行路408号","上海市杨浦区闸殷路1555号102室","上海市杨浦区平凉路1500号"] + def map = [:] + for(def r : rets){ + def location = getLocation(r) + if(location){ + map.put(r, location) + } + } + def titleMap = [0:'C',1:'D',2:'E',3:'F',4:'G',5:'H',6:'I',7:'J',8:'K'] + ExcelUtils.modify("D:\\数据\\上海农商行\\网点与居委分布20230118.xlsx",{row-> + if(row['B'] && !row['B'].contains("地址") && row['B'].contains('齐齐哈尔路')){ + for(def i=0;i OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20221101", "yyyyMMdd") -def endDate = DateUtil.parseDate("20221201", "yyyyMMdd") +def startDate = DateUtil.parseDate("20230101", "yyyyMMdd") +def endDate = DateUtil.parseDate("20230201", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 @@ -38,6 +38,6 @@ while (startDate.before(endDate)) { startDate = nextDate } -ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行.xlsx", data, "202211") +ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, "202301")