From 5b96e0c12bcb1369ccb4e023f4f69902bee63631 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 10 Jan 2023 15:56:54 +0800 Subject: [PATCH 01/30] =?UTF-8?q?=E4=B8=B4=E6=97=B6=E5=AF=BC1=E6=9C=88?= =?UTF-8?q?=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ronghe/ExportDistrictRongHe.groovy | 179 ++++++++++++++++-- 1 file changed, 165 insertions(+), 14 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 892bd5e6c..919bff95a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -29,6 +29,10 @@ class ExportDistrictRongHe extends BaseBatch{ @Override void process(Object args) { + + def queryDate = 202212//DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() + + citys = CcbCommonUtils.getRunningCitys(args) def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") @@ -36,7 +40,8 @@ class ExportDistrictRongHe extends BaseBatch{ //按天导(每天一个文件夹) exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) - if(curDate.toString().endsWith("10")){ + +// if(curDate.toString().endsWith("10")){ exportBigdataCommunityGJRH()// exportBigdataBuildingFJW()// exportBigdataCommfeeGj()// @@ -53,21 +58,26 @@ class ExportDistrictRongHe extends BaseBatch{ exportBigdataCommlayoutpicFjw() exportEvaluateRate() -// //因为融合暂不切库,所以需要加传一张第三方均价表到融合中, 从1月10日开始传输 - def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); - exportThirdPartyPrice2(thirdPartyMonth, new_dists) +// def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() + exportBigdataCityPrice(queryDate) + exportBigdataDistPrice(queryDate) + exportBigdataCommPrice(queryDate) +//// //因为融合暂不切库,所以需要加传一张第三方均价表到融合中, 从1月10日开始传输,长期供 +// def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); +// exportThirdPartyPrice2(thirdPartyMonth, new_dists) +// +// +// //1月10日 新增5张表,导92城,增量数据,2月10日导344城,其中252城所有历史数据,92城增量,3月后可能不用提供数据 +// exportCommunityBasePrice(lastMonth) +// exportCommunityPdescribeGj(lastMonth) +// exportGuideStatistics(lastMonth) +// exportBigdataSupplierStatistics(lastMonth) +// exportBigdataModelingBase(lastMonth) - //1月10日 新增5张表,导92城,增量数据,2月10日导344城,其中252城所有历史数据,92城增量,3月后可能不用提供数据 - exportCommunityBasePrice(lastMonth) - exportCommunityPdescribeGj(lastMonth) - exportGuideStatistics(lastMonth) - exportBigdataSupplierStatistics(lastMonth) - exportBigdataModelingBase(lastMonth) - - } - SSHHelper jschCommand = new SSHHelper(); - System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); +// } +// SSHHelper jschCommand = new SSHHelper(); +// System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); exportOk(curDate) @@ -1141,6 +1151,147 @@ class ExportDistrictRongHe extends BaseBatch{ println "bigdata_modeling_base 上传Ftp完成......" } + def exportBigdataCityPrice(queryDate){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat" + } + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + // def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";// + def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}'";// + + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.YMID + data << r.SALEPRICE?r.SALEPRICE:'' + data << r.RENTPRICE?r.RENTPRICE:'' + if(r.SALECOUNT>0) { + data << r.SALECOUNT + }else{ + data << '' + } + if(r.RENTCOUNT>0){ + data << r.RENTCOUNT + }else{ + data << '' + } + data << r.UPDATETIME + data << r.HOUSEUSE + data << r.SALEBLDGAREA?r.SALEBLDGAREA:'' + data << r.RENTBLDGAREA?r.RENTBLDGAREA:'' + data << r.SALETOTALPRICE?r.SALETOTALPRICE:'' + data << r.RENTTOTALPRICE?r.RENTTOTALPRICE:'' + data << r.DATATYPE + data << r.SDATERANGE + data << r.RDATERANGE + data << r.SOURCECODE + allRows << data + }); + } + + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_CITYPRICE 上传Ftp完成......" + } + + + def exportBigdataDistPrice(queryDate){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat" + } + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID,DISTID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_DISTPRICE where CITYID='${cityCode}'";// and YMID=${queryDate} + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.YMID + data << r.DISTID + data << r.SALEPRICE?r.SALEPRICE:'' + data << r.RENTPRICE?r.RENTPRICE:'' + if(r.SALECOUNT>0) { + data << r.SALECOUNT + }else{ + data << '' + } + if(r.RENTCOUNT>0){ + data << r.RENTCOUNT + }else{ + data << '' + } + data << r.UPDATETIME + data << r.HOUSEUSE + data << r.SALEBLDGAREA?r.SALEBLDGAREA:'' + data << r.RENTBLDGAREA?r.RENTBLDGAREA:'' + data << r.SALETOTALPRICE?r.SALETOTALPRICE:'' + data << r.RENTTOTALPRICE?r.RENTTOTALPRICE:'' + data << r.DATATYPE + data << r.SDATERANGE + data << r.RDATERANGE + data << r.SOURCECODE + allRows << data + }); + } + + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_DISTPRICE 上传Ftp完成......" + } + + + def exportBigdataCommPrice(queryDate){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat" + } + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE where CITYID='${cityCode}' ";// and YMID=${queryDate} + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.YMID + data << r.COMMID + data << r.SALEPRICE?r.SALEPRICE:'' + if(r.SALECOUNT>0) { + data << r.SALECOUNT + }else{ + data << '' + } + data << r.RENTPRICE?r.RENTPRICE:'' + if(r.RENTCOUNT>0){ + data << r.RENTCOUNT + }else{ + data << '' + } + data << r.UPDATETIME + data << r.SALEBLDGAREA?r.SALEBLDGAREA:'' + data << r.RENTBLDGAREA?r.RENTBLDGAREA:'' + data << r.SALETOTALPRICE?r.SALETOTALPRICE:'' + data << r.RENTTOTALPRICE?r.RENTTOTALPRICE:'' + data << r.HOUSEUSE + data << r.SDATERANGE + data << r.RDATERANGE + data << r.DATATYPE + data << r.SOURCECODE + allRows << data + }); + } + + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_COMMPRICE 上传Ftp完成......" + } + def exportBigdataCommlayoutpicFjw(){ From 0d98e1ae838d41c6ecd74d8f24c306c95326e3f9 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 10 Jan 2023 16:20:22 +0800 Subject: [PATCH 02/30] =?UTF-8?q?=E8=A1=8C=E6=83=85=E7=9B=AE=E5=BD=95?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 919bff95a..59e1387c8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -1153,7 +1153,7 @@ class ExportDistrictRongHe extends BaseBatch{ def exportBigdataCityPrice(queryDate){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat" } @@ -1201,7 +1201,7 @@ class ExportDistrictRongHe extends BaseBatch{ def exportBigdataDistPrice(queryDate){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat" } @@ -1248,7 +1248,7 @@ class ExportDistrictRongHe extends BaseBatch{ def exportBigdataCommPrice(queryDate){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat" } From 75503b74f314ace92a6ac2a5cf6bae578aeaba18 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 2 Feb 2023 18:24:16 +0800 Subject: [PATCH 03/30] =?UTF-8?q?=E5=B0=8F=E5=8C=BA=E8=A1=8C=E6=83=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ronghe/ExportDistrictRongHe.groovy | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 59e1387c8..fbf70498c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -38,29 +38,29 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //按天导(每天一个文件夹) - exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) +// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) // if(curDate.toString().endsWith("10")){ - exportBigdataCommunityGJRH()// - exportBigdataBuildingFJW()// - exportBigdataCommfeeGj()// - - exportBigdataCommbuildstruGJ()// - exportBigdataCommhousetypeGJ()// - exportBigdataCommbuildyearGj()// - exportBigdataCommhouseuseGJ()// - exportBigdataPositionGJ()// - exportBigdataDistrictFJW()// - exportBigdataAliasFJW()// - - exportBigdataBuildprojFjw() - exportBigdataCommlayoutpicFjw() - exportEvaluateRate() +// exportBigdataCommunityGJRH()// +// exportBigdataBuildingFJW()// +// exportBigdataCommfeeGj()// +// +// exportBigdataCommbuildstruGJ()// +// exportBigdataCommhousetypeGJ()// +// exportBigdataCommbuildyearGj()// +// exportBigdataCommhouseuseGJ()// +// exportBigdataPositionGJ()// +// exportBigdataDistrictFJW()// +// exportBigdataAliasFJW()// +// +// exportBigdataBuildprojFjw() +// exportBigdataCommlayoutpicFjw() +// exportEvaluateRate() // def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() - exportBigdataCityPrice(queryDate) - exportBigdataDistPrice(queryDate) +// exportBigdataCityPrice(queryDate) +// exportBigdataDistPrice(queryDate) exportBigdataCommPrice(queryDate) //// //因为融合暂不切库,所以需要加传一张第三方均价表到融合中, 从1月10日开始传输,长期供 @@ -79,7 +79,7 @@ class ExportDistrictRongHe extends BaseBatch{ // SSHHelper jschCommand = new SSHHelper(); // System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); - exportOk(curDate) +// exportOk(curDate) } @@ -1161,7 +1161,7 @@ class ExportDistrictRongHe extends BaseBatch{ for(def city : citys) { def cityCode = cityCodeMap.get(city) // def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";// - def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}'";// + def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";// sqlInstance.eachRow(sql,{r-> def data = [] @@ -1208,7 +1208,7 @@ class ExportDistrictRongHe extends BaseBatch{ def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select CITYID,DISTID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_DISTPRICE where CITYID='${cityCode}'";// and YMID=${queryDate} + def sql = "select CITYID,DISTID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_DISTPRICE where CITYID='${cityCode}' and YMID=${queryDate}";// and YMID=${queryDate} sqlInstance.eachRow(sql,{r-> def data = [] data << r.CITYID @@ -1255,7 +1255,7 @@ class ExportDistrictRongHe extends BaseBatch{ def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE where CITYID='${cityCode}' ";// and YMID=${queryDate} + def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE where CITYID='${cityCode}' and YMID=${queryDate} ";// sqlInstance.eachRow(sql,{r-> def data = [] data << r.CITYID @@ -1305,7 +1305,7 @@ class ExportDistrictRongHe extends BaseBatch{ println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......" } - def exportOk(curDate){ + def exportOk(curDate){ // def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok" if(CommonDatautils.isWindow()){ From edef42a8fa095ac3ff21c5df34d1b43018d0ed5f Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 2 Feb 2023 21:54:33 +0800 Subject: [PATCH 04/30] update --- .../ronghe/ExportDistrictRongHe.groovy | 53 +++++++++---------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index fbf70498c..8fe64bdef 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -38,34 +38,33 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //按天导(每天一个文件夹) -// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) + exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) -// if(curDate.toString().endsWith("10")){ -// exportBigdataCommunityGJRH()// -// exportBigdataBuildingFJW()// -// exportBigdataCommfeeGj()// -// -// exportBigdataCommbuildstruGJ()// -// exportBigdataCommhousetypeGJ()// -// exportBigdataCommbuildyearGj()// -// exportBigdataCommhouseuseGJ()// -// exportBigdataPositionGJ()// -// exportBigdataDistrictFJW()// -// exportBigdataAliasFJW()// -// -// exportBigdataBuildprojFjw() -// exportBigdataCommlayoutpicFjw() -// exportEvaluateRate() + if(curDate.toString().endsWith("10")){ + exportBigdataCommunityGJRH()// + exportBigdataBuildingFJW()// + exportBigdataCommfeeGj()// -// def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() -// exportBigdataCityPrice(queryDate) -// exportBigdataDistPrice(queryDate) + exportBigdataCommbuildstruGJ()// + exportBigdataCommhousetypeGJ()// + exportBigdataCommbuildyearGj()// + exportBigdataCommhouseuseGJ()// + exportBigdataPositionGJ()// + exportBigdataDistrictFJW()// + exportBigdataAliasFJW()// + + exportBigdataBuildprojFjw() + exportBigdataCommlayoutpicFjw() + exportEvaluateRate() + + exportBigdataCityPrice(queryDate) + exportBigdataDistPrice(queryDate) exportBigdataCommPrice(queryDate) -//// //因为融合暂不切库,所以需要加传一张第三方均价表到融合中, 从1月10日开始传输,长期供 -// def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); -// exportThirdPartyPrice2(thirdPartyMonth, new_dists) + //因为融合暂不切库,所以需要加传一张第三方均价表到融合中, 从1月10日开始传输,长期供 + def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); + exportThirdPartyPrice2(thirdPartyMonth, new_dists) // // // //1月10日 新增5张表,导92城,增量数据,2月10日导344城,其中252城所有历史数据,92城增量,3月后可能不用提供数据 @@ -75,11 +74,11 @@ class ExportDistrictRongHe extends BaseBatch{ // exportBigdataSupplierStatistics(lastMonth) // exportBigdataModelingBase(lastMonth) -// } -// SSHHelper jschCommand = new SSHHelper(); -// System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); + } + SSHHelper jschCommand = new SSHHelper(); + System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); -// exportOk(curDate) + exportOk(curDate) } From 039924b31c8bc537a0cbad8a1d691f573b8807ae Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 8 Feb 2023 10:57:54 +0800 Subject: [PATCH 05/30] =?UTF-8?q?=E4=B8=B4=E6=97=B6=E5=AF=BC=E5=87=BA?= =?UTF-8?q?=E8=8B=8F=E5=B7=9E=E6=88=BF=E6=BA=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/data/TestData.groovy | 115 ++++++++++++++++ .../ronghe/ExportDistrictRongHe.groovy | 130 +++++++++++++++++- .../com/fangjia/script/chongqing/Des3.groovy | 82 +++++++++++ .../fangjia/script/chongqing/Des3Test.groovy | 100 ++++++++++++++ 4 files changed, 426 insertions(+), 1 deletion(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/TestData.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/Des3.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/Des3Test.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/TestData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/TestData.groovy new file mode 100644 index 000000000..857b62715 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/TestData.groovy @@ -0,0 +1,115 @@ +package com.fangjia.script.ccbevaluation.data + +import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject + +class TestData { + + static main(args) { + def task = new TestData() +// task.run() + task.update() + } + + def update(){ + ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\龙岩1.xlsx", { d -> + def dist = CommonDatautils.findDistrictById(d['A']) + if(dist){ + if(d['E'] && d['G']=="人工"){ + def q = new BasicDBObject() + def baidu = BaiDuApiUtils.geoconv(d['E'].toString().toDouble(), d['F'].toString().toDouble(), 3) + def lng2 = baidu.x + def lat2 = baidu.y + + Long dLat2 = lat2 * 10000000000L + Long dLng2 = lng2 * 10000000000L + + q.put("lat2", dLat2) + q.put("lng2", dLng2) + + Long lng = d['E'].toString().toDouble() * 10000000000L + Long lat = d['F'].toString().toDouble() * 10000000000L + q.put("lng", lng) + q.put("lat", lat) + DbUtils.seaweed.update(new BasicDBObject('_id',d['A']),new BasicDBObject('$set',q)) + } + } + }); + } + + def run(){ + ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\龙岩.xlsx", { d -> + def dist = CommonDatautils.findDistrictById(d['A']) + if(dist){ + def loc = getLocation(dist.city, dist.region, dist.name) + if(loc){ + d['E'] = loc.x + d['F'] = loc.y + println d['C']+" "+loc.x+" "+loc.y + } + } + +// if(!d['E']){ +// def loc = getAmapLocation("龙岩市", d['B'], d['C']) +// if(loc){ +// d['E'] = loc.x +// d['F'] = loc.y +// } +// } + +// if(!d['E']){ +// def dist = CommonDatautils.findDistrictById(d['A']) +// if(dist){ +// def loc = getAmapOldLocation(dist.city, dist.region, d['C']) +// if(loc){ +// d['E'] = loc.x +// d['F'] = loc.y +// } +// } +// +// } + + }); + } + + def getLocation(city, region, name){ + String sql = "select * from community where city='${city}' and region='${region}' and name='${name}' " + def loc = null + ElasticUtils.executeSqlQuery(sql,{ data-> + if(!loc && data.location){ + loc = data.location + } + }) + return loc; + } + + def getAmapLocation(city, region, name){ + String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') " + def loc = null + ElasticUtils.executeSqlQuery(sql,{ data-> + if(!loc && data.location){ + if(name == data.name){ + loc = data.location + } + } + }) + return loc; + } + + def getAmapOldLocation(city, region, name){ + String sql = "select * from amap_poi where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') " + def loc = null + ElasticUtils.executeSqlQuery(sql,{ data-> + if(!loc && data.location){ + if(name == data.name){ + loc = data.location + } + } + }) + return loc; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 8fe64bdef..d65a4e181 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -38,7 +38,7 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //按天导(每天一个文件夹) - exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) + // exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) if(curDate.toString().endsWith("10")){ @@ -75,6 +75,10 @@ class ExportDistrictRongHe extends BaseBatch{ // exportBigdataModelingBase(lastMonth) } + + //2月8号 临时导出苏州一年的数据 + exportBigdataForsale("2022-01-01", new_dists) //现在按updatetime导出 + SSHHelper jschCommand = new SSHHelper(); System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); @@ -1304,6 +1308,130 @@ class ExportDistrictRongHe extends BaseBatch{ println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......" } + def exportBigdataForsale(publishDate, new_dists){ + //["北京","上海"] + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"); + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"] + def index = 0 + citys = ["苏州"] + for(def city : citys){ + def cityCode = cityCodeMap.get(city) + + def allRows = [] + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj " + + "where cityId='${cityCode}' and updatetime>=to_date('${publishDate}','yyyy-MM-dd') and distid not in ${newstr}"; + def rets1 = getHouseDatas(sql1) + allRows.addAll(rets1) + }else{ + def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}' and updatetime>=to_date('${publishDate}','yyyy-MM-dd')"; + allRows = getHouseDatas(sql) + } + if(allRows.size()>0){ + index = index + 1 + if(index == 1){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + } + + } + } + + if(new_dists){//补充被修区域的数据 + def newstr = "('"+new_dists.join("','")+"')" + def sql2 = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH " + + "from bigdata_forsale_gj where distid in ${newstr}" + def rets2 = getHouseDatas(sql2) + if(rets2.size()>0){ + FileUtils.batchCreateFile(rets2, fileName, headers, "utf-8", true); + } + } + println "融合 BIGDATA_FORSALE_GJ 数据文件输出完成......" +// CcbFtpUtils.ftpUpload(null, fileName, curDate) +// println "融合 BIGDATA_FORSALE_GJ 上传Ftp完成......" + } + + def getHouseDatas(sql){ + def allRows = [] + def index = 0 + sqlInstance.eachRow(sql,{r-> + index = index+1 + println r.CITYNAME +" "+index + def data = [] + data << r.CITYID + data << r.FORSALEID + data << r.PROPCODE + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.UNITID + data << r.UNITNO + data << r.UNITNAME + data << r.HOUSEID + data << r.ADDRESS + data << r.BUILDYEAR + data << r.PROPTYPE + data << r.PROPYEARS + data << r.UNITPRICE + data << r.TOTALPRICE + data << r.PAYTYPE + data << r.FLOORNUM + data << r.FLOORCNT + data << r.BUILDTYPE + data << r.HOUSESTRU + data << r.BUILDSTRU + data << r.BUILDAREA + data << r.LIVINGAREA + data << r.HOUSESTYLE + data << r.ROOMCNT + data << r.LIVINGROOMCNT + data << r.KITCHENCNT + data << r.BATHROOMCNT + data << r.HOUSEASPECT + data << r.FITMENT + data << r.IFELEVATOR + data << r.ELEVATORHOUSE + data << r.PUBLISHTIME + data << r.EXPIRETIME + data << r.IFFIRST + data << r.DATASOURCE + data << r.AGENCYID + data << r.AGENCYNAME + data << r.PUBLISHER + data << r.PHONE + data << "" + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.ATTACH + allRows << data +// if(allRows.size()%200000==0){ +// FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); +// allRows.clear() +// } + }); + return allRows + } + + def exportOk(curDate){ // def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/Des3.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/Des3.groovy new file mode 100644 index 000000000..012ddd921 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/Des3.groovy @@ -0,0 +1,82 @@ +package com.fangjia.script.chongqing +import com.sun.org.apache.xml.internal.security.utils.Base64; +import org.apache.commons.lang3.StringUtils; + +import javax.crypto.Cipher; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.DESedeKeySpec; +import javax.crypto.spec.IvParameterSpec; +import java.security.Key; + +class Des3 { + private final static String encoding = "UTF-8"; + + static main(args) { + def task = new Des3() + task.run() + } + def run(){ + try { + String key = "111036369260679051122113"; + String iv = "23235645";//IV length must be 8 bytes long + //加密 + String encryptStr = "Lk+AtbEWoNF0eQNrSsGoNJjRFuvnCaRKaypcAnaRvGpHtc2KmMOMbufut+JDWP5Rq/4qNAWsEch0iZgE5z2CTnAwCjoRTUl0"//encode("test", key, iv); + System.out.println("密文: " + encryptStr); + //解密 + String decryptStr = decode(encryptStr, key, iv); + System.out.println("明文: " + decryptStr); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 方法描述:3DES加密 + * @author hyang + * + * @param plainText 明文 + * @param secretKey 密钥 + * @param iv 加密向量 + * @return String 密文 + * @throws Exception + */ + public static String encode(String plainText, String secretKey, String iv) + throws Exception { + Key deskey = null; + DESedeKeySpec spec = new DESedeKeySpec(secretKey.getBytes()); + SecretKeyFactory keyfactory = SecretKeyFactory.getInstance("desede"); + deskey = keyfactory.generateSecret(spec); + Cipher cipher = Cipher.getInstance("desede/CBC/PKCS5Padding"); + IvParameterSpec ips = new IvParameterSpec(iv.getBytes()); + cipher.init(Cipher.ENCRYPT_MODE, deskey, ips); + byte[] encryptData = cipher.doFinal(plainText.getBytes(encoding)); + return Base64.encode(encryptData); + } + + /** + * 方法描述: 3DES解密 + * @author hyang + * + * @param encryptText 密文 + * @param secretKey 密钥 + * @param iv 加密向量 + * @return String 明文 + * @throws Exception + */ + public static String decode(String encryptText, String secretKey, String iv) + throws Exception { + if (StringUtils.isBlank(encryptText) || StringUtils.isBlank(secretKey)) { + return ""; + } + Key deskey = null; + DESedeKeySpec spec = new DESedeKeySpec(secretKey.getBytes()); + SecretKeyFactory keyfactory = SecretKeyFactory.getInstance("desede"); + deskey = keyfactory.generateSecret(spec); + Cipher cipher = Cipher.getInstance("desede/CBC/NoPadding"); //desede/CBC/NoPadding 无填充 desede/CBC/PKCS5Padding 加密需要填充,解密一般加不加都结果一致 + IvParameterSpec ips = new IvParameterSpec(iv.getBytes()); + cipher.init(Cipher.DECRYPT_MODE, deskey, ips); + byte[] decryptData = cipher.doFinal(Base64.decode(encryptText)); + return new String(decryptData, encoding).trim(); + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/Des3Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/Des3Test.groovy new file mode 100644 index 000000000..a9a798124 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/Des3Test.groovy @@ -0,0 +1,100 @@ +package com.fangjia.script.chongqing + +import com.fangjia.SecretUtils + +import javax.crypto.Cipher +import javax.crypto.SecretKey +import javax.crypto.spec.SecretKeySpec + +class Des3Test { + //定义加密算法,有DES、DESede(即3DES)、Blowfish + private static final String Algorithm = "DESede"; + private static final String PASSWORD_CRYPT_KEY = "111036369260679051122113"; + + + /** + * 加密方法 + * @param src 源数据的字节数组 + * @return + */ + public static byte[] encryptMode(byte[] src) { + try { + SecretKey deskey = new SecretKeySpec(build3DesKey(PASSWORD_CRYPT_KEY), Algorithm); //生成密钥 + Cipher c1 = Cipher.getInstance(Algorithm); //实例化负责加密/解密的Cipher工具类 + c1.init(Cipher.ENCRYPT_MODE, deskey); //初始化为加密模式 + return c1.doFinal(src); + } catch (java.security.NoSuchAlgorithmException e1) { + e1.printStackTrace(); + } catch (javax.crypto.NoSuchPaddingException e2) { + e2.printStackTrace(); + } catch (java.lang.Exception e3) { + e3.printStackTrace(); + } + return null; + } + + + /** + * 解密函数 + * @param src 密文的字节数组 + * @return + */ + public static byte[] decryptMode(byte[] src) { + try { + SecretKey deskey = new SecretKeySpec(build3DesKey(PASSWORD_CRYPT_KEY), Algorithm); + Cipher c1 = Cipher.getInstance(Algorithm); + c1.init(Cipher.DECRYPT_MODE, deskey); //初始化为解密模式 + return c1.doFinal(src); + } catch (java.security.NoSuchAlgorithmException e1) { + e1.printStackTrace(); + } catch (javax.crypto.NoSuchPaddingException e2) { + e2.printStackTrace(); + } catch (java.lang.Exception e3) { + e3.printStackTrace(); + } + return null; + } + + + /* + * 根据字符串生成密钥字节数组 + * @param keyStr 密钥字符串 + * @return + * @throws UnsupportedEncodingException + */ + public static byte[] build3DesKey(String keyStr) throws UnsupportedEncodingException{ + byte[] key = new byte[24]; //声明一个24位的字节数组,默认里面都是0 + byte[] temp = keyStr.getBytes("UTF-8"); //将字符串转成字节数组 + + /* + * 执行数组拷贝 + * System.arraycopy(源数组,从源数组哪里开始拷贝,目标数组,拷贝多少位) + */ + if(key.length > temp.length){ + //如果temp不够24位,则拷贝temp数组整个长度的内容到key数组中 + System.arraycopy(temp, 0, key, 0, temp.length); + }else{ + //如果temp大于24位,则拷贝temp数组24个长度的内容到key数组中 + System.arraycopy(temp, 0, key, 0, key.length); + } + return key; + } + + /** + * @param args + */ + public static void main(String[] args) { + String msg = "3DES加密解密案例"; + System.out.println("【加密前】:" + msg); + + //加密 +// byte[] secretArr = Des3Test.encryptMode(msg.getBytes()); +// System.out.println("【加密后】:" + new String(secretArr)); + + String s = "Lk+AtbEWoNF0eQNrSsGoNJjRFuvnCaRKaypcAnaRvGpHtc2KmMOMbufut+JDWP5Rq/4qNAWsEch0iZgE5z2CTnAwCjoRTUl0" + + //解密 + byte[] myMsgArr = Des3Test.decryptMode(s.getBytes("UTF-8")); + System.out.println("【解密后】:" + new String(myMsgArr)); + } +} From 3be00b6aaac94303b258a585a73b42eb5e41c2a9 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 8 Feb 2023 11:19:02 +0800 Subject: [PATCH 06/30] update --- .../ronghe/BigdataCitypriceBatch.groovy | 6 ++--- .../ronghe/ExportDistrictRongHe.groovy | 7 +++-- .../fuzhou/DistrictPriceExportBatch.groovy | 2 +- .../TestCrawlTotalCount.groovy | 4 +-- .../fangjia/script/util/BaiduMapUtils.groovy | 27 +++++++++---------- .../yinjihaun/amap/GaoDeApiUtils.groovy | 2 +- 6 files changed, 22 insertions(+), 26 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index 2d248a235..e3b08e9e1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -27,7 +27,7 @@ class BigdataCitypriceBatch extends BaseBatch{ public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) - def curRunDate = 202212//DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger(); + def curRunDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger(); static main(args) { runBatch(new BigdataCitypriceBatch(), args) @@ -36,8 +36,8 @@ class BigdataCitypriceBatch extends BaseBatch{ @Override void process(Object args) { def citys = CcbCommonUtils.getRunningCitys(args) - def startDate = 20221201//DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger(); - def endDate = 20221231//DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger(); + def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger(); + def endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger(); def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() cityRun(citys, startDate, endDate, cityCodeDataMap) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index d65a4e181..33a6b05c2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -30,7 +30,7 @@ class ExportDistrictRongHe extends BaseBatch{ @Override void process(Object args) { - def queryDate = 202212//DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() + def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() citys = CcbCommonUtils.getRunningCitys(args) @@ -38,7 +38,7 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //按天导(每天一个文件夹) - // exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) + exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) if(curDate.toString().endsWith("10")){ @@ -77,7 +77,7 @@ class ExportDistrictRongHe extends BaseBatch{ } //2月8号 临时导出苏州一年的数据 - exportBigdataForsale("2022-01-01", new_dists) //现在按updatetime导出 +// exportBigdataForsale("2022-01-01", new_dists) //现在按updatetime导出 SSHHelper jschCommand = new SSHHelper(); System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); @@ -1317,7 +1317,6 @@ class ExportDistrictRongHe extends BaseBatch{ } def headers = []//["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"] def index = 0 - citys = ["苏州"] for(def city : citys){ def cityCode = cityCodeMap.get(city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy index 8d77112a6..dd6f1e2dc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy @@ -33,7 +33,7 @@ class DistrictPriceExportBatch extends BaseBatch{ static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com") - def specialDistsPriceMap = ["津泰新村7号、9-17号":53620,"津泰新村14号":76230,"津泰新村1-6号、8号":32580] + def specialDistsPriceMap = ["津泰新村7号、9-13号、15-17号":53620,"津泰新村14号":76230,"津泰新村1-6号、8号":32580] static main(args) { runBatch(new DistrictPriceExportBatch(), args) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index df803e640..2d9309dae 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] for(def source :sources){ - def count1 = findTotalCount(source, 202211); - def count2 = findTotalCount(source, 202212); + def count1 = findTotalCount(source, 202212); + def count2 = findTotalCount(source, 202301); println source +" "+count1 +" "+count2 } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/BaiduMapUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/BaiduMapUtils.groovy index 3bfe575e9..2eb3fcfed 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/BaiduMapUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/BaiduMapUtils.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.util +import com.fangjia.core.util.HttpRequest import com.fangjia.core.util.RandomUtil import com.fangjia.script.addrepo.InvokeHttpRequest; import groovy.json.JsonSlurper @@ -103,24 +104,20 @@ class BaiduMapUtils { int recount = 0; while(!stop && recount<3) { try{ - def ret = ABuYunUtil.doGet(url, null, null, "utf-8"); - if (ret) { - JSONObject jsonObj = JSONObject.fromObject(ret); - - if(jsonObj.getInt("status") == 121){ - println "key over" - changeAK() - recount++; + HttpRequest request = new HttpRequest(); + try{ + String result = request.get(url,null,"UTF-8"); + JSONObject json = JSONObject.fromObject(result); + if(json.getInt("status") >= 300){ + changeAK(); + json = searchByAddress(address); }else{ - def result = jsonObj.result; - if(result){ - stop = true - return result; - }else{ - stop = true - } + return json.result } + }catch (Exception e){ + e.printStackTrace(); } + return null; }catch (Exception e){ recount++; println e.getMessage() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy index fdcf818c3..4d301536b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy @@ -12,7 +12,7 @@ import org.jsoup.Jsoup import java.util.zip.GZIPInputStream class GaoDeApiUtils { - def static GAO_DE_AK = "90010d53a16ce9df544acab297645106" + def static GAO_DE_AK = "c6fcd4bfbc40d73ee762b8115dca3390" /** * 经纬度转地址
From f6bf27b743d8bacdf680f311ea424f56fb168d6d Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 8 Feb 2023 14:33:34 +0800 Subject: [PATCH 07/30] =?UTF-8?q?=E5=AF=BC=E8=8B=8F=E5=B7=9E=E6=88=BF?= =?UTF-8?q?=E6=BA=90=20=E5=87=8F=E5=B0=91=E4=B8=80=E4=B8=AA=E5=AD=97?= =?UTF-8?q?=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/ronghe/ExportDistrictRongHe.groovy | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 33a6b05c2..d43bfa2cb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -77,7 +77,7 @@ class ExportDistrictRongHe extends BaseBatch{ } //2月8号 临时导出苏州一年的数据 -// exportBigdataForsale("2022-01-01", new_dists) //现在按updatetime导出 + exportBigdataForsale("2022-01-01", new_dists) //现在按updatetime导出 SSHHelper jschCommand = new SSHHelper(); System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); @@ -1310,10 +1310,11 @@ class ExportDistrictRongHe extends BaseBatch{ def exportBigdataForsale(publishDate, new_dists){ //["北京","上海"] + citys = ["苏州"] def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"); - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat" } def headers = []//["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"] def index = 0 @@ -1420,7 +1421,7 @@ class ExportDistrictRongHe extends BaseBatch{ data << r.CREATETIME data << r.UPDATETIME data << r.SOURCECODE - data << r.ATTACH +// data << r.ATTACH //融合房源没有这个字段 allRows << data // if(allRows.size()%200000==0){ // FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); From b550e3b1f244403b06edf0da55ae79cf73127230 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 9 Feb 2023 17:01:58 +0800 Subject: [PATCH 08/30] =?UTF-8?q?=E8=9E=8D=E5=90=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/CcbCommonUtils.groovy | 3 +- .../ronghe/ExportDistrictRongHe.groovy | 13 +++--- .../script/wenyan/temp/DongGuanBank.groovy | 43 +++++++++++++++++++ 3 files changed, 52 insertions(+), 7 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/DongGuanBank.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 24d06b991..908f4adf5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -16,7 +16,8 @@ import groovy.sql.Sql class CcbCommonUtils { // - public static def citys_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"] +// public static def citys_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"] + public static def citys_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"] public static def citys_92 = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"] public static def citys_252 = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index d43bfa2cb..efcbda2f1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -38,7 +38,7 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //按天导(每天一个文件夹) - exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) +// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) if(curDate.toString().endsWith("10")){ @@ -77,7 +77,7 @@ class ExportDistrictRongHe extends BaseBatch{ } //2月8号 临时导出苏州一年的数据 - exportBigdataForsale("2022-01-01", new_dists) //现在按updatetime导出 + exportBigdataForsale("2022-07-01", new_dists) //现在按updatetime导出 SSHHelper jschCommand = new SSHHelper(); System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); @@ -1309,8 +1309,6 @@ class ExportDistrictRongHe extends BaseBatch{ } def exportBigdataForsale(publishDate, new_dists){ - //["北京","上海"] - citys = ["苏州"] def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"); def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ @@ -1320,6 +1318,9 @@ class ExportDistrictRongHe extends BaseBatch{ def index = 0 for(def city : citys){ def cityCode = cityCodeMap.get(city) + if(city=="苏州"){ + continue + } def allRows = [] if(new_dists){ @@ -1359,8 +1360,8 @@ class ExportDistrictRongHe extends BaseBatch{ } } println "融合 BIGDATA_FORSALE_GJ 数据文件输出完成......" -// CcbFtpUtils.ftpUpload(null, fileName, curDate) -// println "融合 BIGDATA_FORSALE_GJ 上传Ftp完成......" + CcbFtpUtils.ftpUpload("rh",fileName, curDate) + println "融合 BIGDATA_FORSALE_GJ 上传Ftp完成......" } def getHouseDatas(sql){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/DongGuanBank.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/DongGuanBank.groovy new file mode 100644 index 000000000..6e992809d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/DongGuanBank.groovy @@ -0,0 +1,43 @@ +package com.fangjia.script.wenyan.temp + +import com.fangjia.core.Constants +import com.fangjia.core.util.DateUtil +import com.fangjia.core.util.Mongo3Utils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.client.MongoCollection +import net.sf.json.JSONObject +import org.bson.Document + +import static com.mongodb.client.model.Filters.* + +MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") +def startDate = DateUtil.parseDate("20221101", "yyyyMMdd") +def endDate = DateUtil.parseDate("20221201", "yyyyMMdd") +Calendar calendar = Calendar.getInstance() +calendar.setTime(startDate) +def count = 0 +def hashSet = [] +def data = [["编号", "公司名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "返回参数"]] +while (startDate.before(endDate)) { + calendar.add(Calendar.DATE, 1) + def nextDate = calendar.getTime() + count++ + OPEN_URI_LOG.find(and(eq("user_id", "1007713"), eq("request_uri", "/property/evaluate"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc -> + def result = JSONObject.fromObject(doc.res_map.result) +// if (!hashSet.contains(result)) { + def item = [doc._id.toString(), "东莞银行总行", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()] + data << item +// hashSet << result +// } else { +// println("已存在") +// } + } + if (count % 7 == 0) { + hashSet.clear() + } + + startDate = nextDate +} +ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行.xlsx", data, "202211") + + From 5e779c54b3d9b1e4e2358328a373dd05d2930f7c Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 9 Feb 2023 22:28:49 +0800 Subject: [PATCH 09/30] up --- .../ccbevaluation/ronghe/ExportDistrictRongHe.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index efcbda2f1..2f34573dd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -38,7 +38,7 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //按天导(每天一个文件夹) -// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) + exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) if(curDate.toString().endsWith("10")){ @@ -77,7 +77,7 @@ class ExportDistrictRongHe extends BaseBatch{ } //2月8号 临时导出苏州一年的数据 - exportBigdataForsale("2022-07-01", new_dists) //现在按updatetime导出 +// exportBigdataForsale("2022-07-01", new_dists) //现在按updatetime导出 SSHHelper jschCommand = new SSHHelper(); System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); @@ -1197,7 +1197,7 @@ class ExportDistrictRongHe extends BaseBatch{ } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); - CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + CcbFtpUtils.ftpUpload("rh",fileName, curDate) println "BIGDATA_CITYPRICE 上传Ftp完成......" } @@ -1244,7 +1244,7 @@ class ExportDistrictRongHe extends BaseBatch{ } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); - CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + CcbFtpUtils.ftpUpload("rh",fileName, curDate) println "BIGDATA_DISTPRICE 上传Ftp完成......" } @@ -1291,7 +1291,7 @@ class ExportDistrictRongHe extends BaseBatch{ } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); - CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + CcbFtpUtils.ftpUpload("rh",fileName, curDate) println "BIGDATA_COMMPRICE 上传Ftp完成......" } @@ -1309,7 +1309,7 @@ class ExportDistrictRongHe extends BaseBatch{ } def exportBigdataForsale(publishDate, new_dists){ - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat" From 54e650f6369571d4153c7a5d44343df6a4f9dc1f Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 10 Feb 2023 09:54:35 +0800 Subject: [PATCH 10/30] =?UTF-8?q?=E8=9E=8D=E5=90=88-=E5=AF=BC=E4=BC=B0?= =?UTF-8?q?=E5=80=BC=E7=9B=B8=E5=85=B3=E7=9A=84=E8=A1=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ronghe/ExportDistrictRongHe.groovy | 66 +++++++++---------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 2f34573dd..7c919169c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -38,51 +38,51 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //按天导(每天一个文件夹) - exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) +// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) - if(curDate.toString().endsWith("10")){ - exportBigdataCommunityGJRH()// - exportBigdataBuildingFJW()// - exportBigdataCommfeeGj()// - - exportBigdataCommbuildstruGJ()// - exportBigdataCommhousetypeGJ()// - exportBigdataCommbuildyearGj()// - exportBigdataCommhouseuseGJ()// - exportBigdataPositionGJ()// - exportBigdataDistrictFJW()// + if(curDate.toString().endsWith("11")){ +// exportBigdataCommunityGJRH()// +// exportBigdataBuildingFJW()// +// exportBigdataCommfeeGj()// +// +// exportBigdataCommbuildstruGJ()// +// exportBigdataCommhousetypeGJ()// +// exportBigdataCommbuildyearGj()// +// exportBigdataCommhouseuseGJ()// +// exportBigdataPositionGJ()// +// exportBigdataDistrictFJW()// exportBigdataAliasFJW()// - exportBigdataBuildprojFjw() - exportBigdataCommlayoutpicFjw() - exportEvaluateRate() - - exportBigdataCityPrice(queryDate) - exportBigdataDistPrice(queryDate) - exportBigdataCommPrice(queryDate) - - //因为融合暂不切库,所以需要加传一张第三方均价表到融合中, 从1月10日开始传输,长期供 - def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); - exportThirdPartyPrice2(thirdPartyMonth, new_dists) +// exportBigdataBuildprojFjw() +// exportBigdataCommlayoutpicFjw() +// exportEvaluateRate() +// +// exportBigdataCityPrice(queryDate) +// exportBigdataDistPrice(queryDate) +// exportBigdataCommPrice(queryDate) +// +// //因为融合暂不切库,所以需要加传一张第三方均价表到融合中, 从1月10日开始传输,长期供 +// def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); +// exportThirdPartyPrice2(thirdPartyMonth, new_dists) // // -// //1月10日 新增5张表,导92城,增量数据,2月10日导344城,其中252城所有历史数据,92城增量,3月后可能不用提供数据 -// exportCommunityBasePrice(lastMonth) -// exportCommunityPdescribeGj(lastMonth) -// exportGuideStatistics(lastMonth) -// exportBigdataSupplierStatistics(lastMonth) -// exportBigdataModelingBase(lastMonth) + //1月10日 新增5张表,导92城,增量数据,2月10日导344城,其中252城所有历史数据,92城增量,3月后可能不用提供数据 + exportCommunityBasePrice(lastMonth) + exportCommunityPdescribeGj(lastMonth) + exportGuideStatistics(lastMonth) + exportBigdataSupplierStatistics(lastMonth) + exportBigdataModelingBase(lastMonth) } //2月8号 临时导出苏州一年的数据 // exportBigdataForsale("2022-07-01", new_dists) //现在按updatetime导出 - SSHHelper jschCommand = new SSHHelper(); - System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); - - exportOk(curDate) +// SSHHelper jschCommand = new SSHHelper(); +// System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); +// +// exportOk(curDate) } From ba9322e206ba373e023c39e54c29913a358acb81 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 10 Feb 2023 16:41:04 +0800 Subject: [PATCH 11/30] =?UTF-8?q?=E8=BF=98=E5=8E=9F=E8=9E=8D=E5=90=88?= =?UTF-8?q?=E5=AF=BC=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ronghe/ExportDistrictRongHe.groovy | 67 +++++++++---------- .../gj/paimai/BeiKeCrawlByChrome.groovy | 41 ++++++------ .../gj/paimai/LianjiaCrawlByChrome.groovy | 38 ++++++----- 3 files changed, 76 insertions(+), 70 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 7c919169c..8215007e5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -38,35 +38,34 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //按天导(每天一个文件夹) -// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) + exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) + if(curDate.toString().endsWith("10")){ + exportBigdataCommunityGJRH()// + exportBigdataBuildingFJW()// + exportBigdataCommfeeGj()// - if(curDate.toString().endsWith("11")){ -// exportBigdataCommunityGJRH()// -// exportBigdataBuildingFJW()// -// exportBigdataCommfeeGj()// -// -// exportBigdataCommbuildstruGJ()// -// exportBigdataCommhousetypeGJ()// -// exportBigdataCommbuildyearGj()// -// exportBigdataCommhouseuseGJ()// -// exportBigdataPositionGJ()// -// exportBigdataDistrictFJW()// + exportBigdataCommbuildstruGJ()// + exportBigdataCommhousetypeGJ()// + exportBigdataCommbuildyearGj()// + exportBigdataCommhouseuseGJ()// + exportBigdataPositionGJ()// + exportBigdataDistrictFJW()// exportBigdataAliasFJW()// -// exportBigdataBuildprojFjw() -// exportBigdataCommlayoutpicFjw() -// exportEvaluateRate() -// -// exportBigdataCityPrice(queryDate) -// exportBigdataDistPrice(queryDate) -// exportBigdataCommPrice(queryDate) -// -// //因为融合暂不切库,所以需要加传一张第三方均价表到融合中, 从1月10日开始传输,长期供 -// def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); -// exportThirdPartyPrice2(thirdPartyMonth, new_dists) -// -// + exportBigdataBuildprojFjw() + exportBigdataCommlayoutpicFjw() + exportEvaluateRate() + + exportBigdataCityPrice(queryDate) + exportBigdataDistPrice(queryDate) + exportBigdataCommPrice(queryDate) + + //因为融合暂不切库,所以需要加传一张第三方均价表到融合中, 从1月10日开始传输,长期供 + def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); + exportThirdPartyPrice2(thirdPartyMonth, new_dists) + + //1月10日 新增5张表,导92城,增量数据,2月10日导344城,其中252城所有历史数据,92城增量,3月后可能不用提供数据 exportCommunityBasePrice(lastMonth) exportCommunityPdescribeGj(lastMonth) @@ -74,15 +73,15 @@ class ExportDistrictRongHe extends BaseBatch{ exportBigdataSupplierStatistics(lastMonth) exportBigdataModelingBase(lastMonth) + //2月8号 临时导出苏州一年的数据 +// exportBigdataForsale("2022-07-01", new_dists) //现在按updatetime导出 + } - //2月8号 临时导出苏州一年的数据 -// exportBigdataForsale("2022-07-01", new_dists) //现在按updatetime导出 + SSHHelper jschCommand = new SSHHelper(); + System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); -// SSHHelper jschCommand = new SSHHelper(); -// System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); -// -// exportOk(curDate) + exportOk(curDate) } @@ -1164,7 +1163,7 @@ class ExportDistrictRongHe extends BaseBatch{ for(def city : citys) { def cityCode = cityCodeMap.get(city) // def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";// - def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";// + def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE_FJW where CITYID='${cityCode}' and YMID=${queryDate}";// sqlInstance.eachRow(sql,{r-> def data = [] @@ -1211,7 +1210,7 @@ class ExportDistrictRongHe extends BaseBatch{ def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select CITYID,DISTID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_DISTPRICE where CITYID='${cityCode}' and YMID=${queryDate}";// and YMID=${queryDate} + def sql = "select CITYID,DISTID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_DISTPRICE_FJW where CITYID='${cityCode}' and YMID=${queryDate}";// and YMID=${queryDate} sqlInstance.eachRow(sql,{r-> def data = [] data << r.CITYID @@ -1258,7 +1257,7 @@ class ExportDistrictRongHe extends BaseBatch{ def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE where CITYID='${cityCode}' and YMID=${queryDate} ";// + def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE_FJW where CITYID='${cityCode}' and YMID=${queryDate} ";// sqlInstance.eachRow(sql,{r-> def data = [] data << r.CITYID diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 24f03c7cc..f641c214c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -251,26 +251,29 @@ public class BeiKeCrawlByChrome { detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd") } detail.source = "贝壳找房" - def detailUrl = e.select(".title a").attr("href") - detail.url = detailUrl - def dbo = findTrade(detail) - if (!dbo) { - def houseInfo = e.select(".houseInfo").text() - def houseInfos = houseInfo.split("\\|") - detail.direction = houseInfos[0] - detail.fitment = houseInfos[1] - def floorStr = e.select(".positionInfo").text() - if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { - floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) - if (floorStr && floorStr.isNumber()) { - detail.height = floorStr.toInteger() - } - } - detail.create_date = new Date() - detail.m_date = new Date() - println detail - DEAL_PRICE_COLL.save(detail) + if(detail.total_price){ + def detailUrl = e.select(".title a").attr("href") + detail.url = detailUrl + def dbo = findTrade(detail) + if (!dbo) { + def houseInfo = e.select(".houseInfo").text() + def houseInfos = houseInfo.split("\\|") + detail.direction = houseInfos[0] + detail.fitment = houseInfos[1] + def floorStr = e.select(".positionInfo").text() + if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { + floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) + if (floorStr && floorStr.isNumber()) { + detail.height = floorStr.toInteger() + } + } + + detail.create_date = new Date() + detail.m_date = new Date() + println detail + DEAL_PRICE_COLL.save(detail) + } } } }catch(Exception ex){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 0ecdddeb1..e82b8bc45 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -328,25 +328,29 @@ public class LianjiaCrawlByChrome { } detail.source = "链家在线" detail.url = e.select(".title a").attr("href") - def dbo = findTrade(detail) - if (!dbo) { - def houseInfo = e.select(".houseInfo").text() - def houseInfos = houseInfo.split("\\|") - detail.direction = houseInfos[0] - detail.fitment = houseInfos[1] - def floorStr = e.select(".positionInfo").text() - if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { - floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) - if (floorStr && floorStr.isNumber()) { - detail.height = floorStr.toInteger() - } - } - detail.create_date = new Date() - detail.m_date = new Date() - println detail - DEAL_PRICE_COLL.save(detail) + if(detail.total_price){ + def dbo = findTrade(detail) + if (!dbo) { + def houseInfo = e.select(".houseInfo").text() + def houseInfos = houseInfo.split("\\|") + detail.direction = houseInfos[0] + detail.fitment = houseInfos[1] + def floorStr = e.select(".positionInfo").text() + if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { + floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) + if (floorStr && floorStr.isNumber()) { + detail.height = floorStr.toInteger() + } + } + + detail.create_date = new Date() + detail.m_date = new Date() + println detail + DEAL_PRICE_COLL.save(detail) + } } + } } }catch(Exception ex){ From 0d9396305d0de89a6f4cd0c16bbf3e148f46fc73 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 17 Feb 2023 09:30:58 +0800 Subject: [PATCH 12/30] update --- .../BigdataCommunityBatch.groovy | 62 ++++++++++++++++++- 1 file changed, 59 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 27ed0ad49..efdf794af 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -12,6 +12,7 @@ import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.OpenApiClient +import groovy.sql.Sql import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch @@ -30,6 +31,9 @@ class BigdataCommunityBatch extends BaseBatch{ public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) + def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/postgres?stringtype=unspecified", + "postgres", "postgres", "org.postgresql.Driver") + static main(args) { runBatch(new BigdataCommunityBatch(), args); } @@ -37,7 +41,7 @@ class BigdataCommunityBatch extends BaseBatch{ @Override public void process(Object args) { def citys = CcbCommonUtils.getRunningCitys(args) - +// citys = ["文昌"] def rows = [] def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途", @@ -71,7 +75,9 @@ class BigdataCommunityBatch extends BaseBatch{ allDatas << headers allDatas.addAll(rows) FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); - println "txt 生成" + println "txt 生成->导Oracle" + + saveToPgSql(rows) // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据") } @@ -169,7 +175,7 @@ class BigdataCommunityBatch extends BaseBatch{ ret << CcbCodeUtils.parseBuildYear(dist.finish_building_date) ret << CcbCodeUtils.parseHouseUse(dist.estate_type2) ret << dist.household_count - ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd") + ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd").toInteger() ret << dist.status ret << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss") ret << "pa101" @@ -222,4 +228,54 @@ class BigdataCommunityBatch extends BaseBatch{ } return ret; } + + def saveToPgSql(rows){ + for(def r : rows) { + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ + r[i] = null + } + } + r << null + r << '2023-02-28' + r << null + r << '2999-12-31' + r << null + r << null + r << null + } + + println "start: "+new Date() + def sql = "insert into warehouse.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" +// def sql = "insert into base.t0831_bigdata_community_gj_h values " + def index = 0 + sqlInstance.withBatch(20000, sql){ ps-> + for(def r : rows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], + r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], null, '2023-02-28', null, '2999-12-31', null, null, null) + ps.toString() + } + } + for(def r : rows){ + index ++ + if(index == 1){ + sql = sql + "("+r[0].toString()+","+r[1].toString()+","+r[2].toString()+")" + }else{ + sql = sql + ",("+")" + } + } +// sqlInstance.withBatch(50000, sql){ ps-> +// def index = 0 +// for(def r : rows){ +// index++ +// for(def i =0 ;i Date: Fri, 17 Feb 2023 10:21:49 +0800 Subject: [PATCH 13/30] update --- .../fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index efdf794af..1580dda9b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -31,7 +31,7 @@ class BigdataCommunityBatch extends BaseBatch{ public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) - def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/postgres?stringtype=unspecified", + def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified", "postgres", "postgres", "org.postgresql.Driver") static main(args) { @@ -246,7 +246,7 @@ class BigdataCommunityBatch extends BaseBatch{ } println "start: "+new Date() - def sql = "insert into warehouse.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" // def sql = "insert into base.t0831_bigdata_community_gj_h values " def index = 0 sqlInstance.withBatch(20000, sql){ ps-> From aa5ada6ba684802cf804f20ed76ae7e9c07ac711 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 17 Feb 2023 13:09:27 +0800 Subject: [PATCH 14/30] update --- .../groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy index 32c4d1f34..1dd114d8a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy @@ -420,8 +420,8 @@ class HouseExportBatch extends BaseBatch{ row << seawater.name row << seawater.url -// row << seawater.p_date - row << DateUtils.getRandomDate("2022-06-01", "2022-06-30") + row << seawater.p_date +// row << DateUtils.getRandomDate("2022-06-01", "2022-06-30") rets << row } } From b592fa85273914a71d2e8e92c3710854176e9461 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Fri, 17 Feb 2023 13:13:07 +0800 Subject: [PATCH 15/30] =?UTF-8?q?=E5=8C=B9=E9=85=8D=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=E8=87=AA=E7=84=B6=E6=9D=91=E5=8C=B9=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/AddressController.java | 106 +++++++++++++++++- 1 file changed, 104 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index 2ae672241..0167259de 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -24,6 +24,7 @@ import com.fangjia.open.model.*; import net.sf.json.JSONArray; import net.sf.json.JSONObject; +import org.apache.commons.codec.binary.Base64; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.lucene.index.Term; @@ -85,7 +86,6 @@ import com.fangjia.service.mongo.fangjia.ServerAreaService; import com.fangjia.service.open.WaitCrawlKeywordsService; import com.fangjia.service.sql.FjtEvaluationService; import com.fangjia.service.web.AddressService; -import org.sothis.web.mvc.annotation.Param; @Authority(role = Role.USER) public class AddressController { @@ -116,6 +116,8 @@ public class AddressController { private static final String GRADE_CONSUME = "消费评分"; public final static OpenConfig CONFIG = OpenConfig.getConfig(); + private static final String ES_USER = "developer"; + private static final String ES_PASSWORD = "5e8edd851d2fdfbd7415232c67367cc3"; /** * @deprecated see parse2Action @@ -522,7 +524,53 @@ public class AddressController { MatchDistrictResult matchResult = addressMatchProxyService.checkMatchDistrictCorrect(param); jsonResult.put("matchResult", matchResult); } else if (jsonResult != null) { - jsonResult.put("matchResult", new MatchDistrictResult()); + // 判断是否为自然村 + boolean matchedVillage = false; + if (jsonResult.get("searchAddress") != null) { + JSONObject searchAddress = (JSONObject) jsonResult.get("searchAddress"); + String city = searchAddress.getString("city"); + String region = searchAddress.getString("district"); + String name = searchAddress.getString("keywords"); + String township = searchAddress.getString("township"); + if (StringUtils.isNotBlank(city) && StringUtils.isNotBlank(region) && StringUtils.isNotBlank(name)) { + List villages = findVillage(city, region, name); + if (CollectionUtils.isNotEmpty(villages)) { + if (villages.size() == 1 || StringUtils.isBlank(township)) { + JSONObject village = villages.get(0).getJSONObject("_source"); + jsonResult.put("id", villages.get(0).getString("_id")); + jsonResult.put("province", village.getString("province")); + jsonResult.put("city", village.getString("city")); + jsonResult.put("district", village.getString("region")); + jsonResult.put("township", village.getString("township")); + jsonResult.put("name", village.getString("name")); + jsonResult.put("lng", village.getJSONObject("location").getDouble("lon")); + jsonResult.put("lat", village.getJSONObject("location").getDouble("lat")); + jsonResult.put("propertyType", "自然村"); + matchedVillage = true; + } else { + for (JSONObject source : villages) { + JSONObject village = source.getJSONObject("_source"); + if (village.getString("township").contains(township) || township.contains(village.getString("township"))) { + jsonResult.put("id", source.getString("_id")); + jsonResult.put("province", village.getString("province")); + jsonResult.put("city", village.getString("city")); + jsonResult.put("district", village.getString("region")); + jsonResult.put("township", village.getString("township")); + jsonResult.put("name", village.getString("name")); + jsonResult.put("lng", village.getJSONObject("location").getDouble("lon")); + jsonResult.put("lat", village.getJSONObject("location").getDouble("lat")); + jsonResult.put("propertyType", "自然村"); + matchedVillage = true; + break; + } + } + } + } + } + } + if (matchedVillage == false) { + jsonResult.put("matchResult", new MatchDistrictResult()); + } } resultModel.setResult(json.get("result")); mav.setModel(resultModel); @@ -918,6 +966,60 @@ public class AddressController { return ret; } + private JSONObject addTerm(String name, String value){ + JSONObject term = new JSONObject(); + JSONObject match = new JSONObject(); + match.put(name, value); + term.put("term", match); + return term; + } + + /*** + * 获取自然村 + * @param city + * @param region + * @param name + * @return + */ + private List findVillage(String city, String region, String name) { + JSONObject reqBody = new JSONObject(); + reqBody.put("from", 0); + reqBody.put("size", 5); + JSONObject query = new JSONObject(); + JSONObject bool = new JSONObject(); + JSONArray must = new JSONArray(); + must.add(addTerm("city", city)); + must.add(addTerm("region", region)); + must.add(addTerm("name", name)); + bool.put("must", must); + query.put("bool", bool); + reqBody.put("query", query); + Base64 token = new Base64(); + String authenticationEncoding = token.encodeAsString((""+ES_USER+":"+ES_PASSWORD+"").getBytes()); + HttpRequest request = new HttpRequest(); + Map headers = new HashMap(); + headers.put("Content-Type", "application/json"); + headers.put("Authorization", "Basic "+authenticationEncoding+""); + try { + String response = request.postWithBody("http://192.168.0.45:9200/village/_search", reqBody, headers, "UTF-8"); + JSONObject json = JSONObject.fromObject(response); + if(!json.containsKey("hits") || !json.getJSONObject("hits").containsKey("hits")){ + return Collections.emptyList(); + } + JSONArray hits = json.getJSONObject("hits").getJSONArray("hits"); + List results = new ArrayList(); + for(int i=0 ;i < hits.size(); i++){ + JSONObject jsonObject = hits.getJSONObject(i); + results.add(jsonObject); + } + return results; + + } catch (HttpRequestException e) { + e.printStackTrace(); + } + return Collections.emptyList(); + } + /** * 加载测试数据 * From abe48b48f3618faac8f9607c05ec49239927c500 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Fri, 17 Feb 2023 13:24:33 +0800 Subject: [PATCH 16/30] =?UTF-8?q?=E5=8C=B9=E9=85=8D=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=E8=87=AA=E7=84=B6=E6=9D=91=E5=8C=B9=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/AddressController.java | 80 ++++++++++--------- 1 file changed, 42 insertions(+), 38 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index 0167259de..3902aaa98 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -87,7 +87,7 @@ import com.fangjia.service.open.WaitCrawlKeywordsService; import com.fangjia.service.sql.FjtEvaluationService; import com.fangjia.service.web.AddressService; -@Authority(role = Role.USER) +//@Authority(role = Role.USER) public class AddressController { private AddressService addressService; @@ -492,7 +492,7 @@ public class AddressController { * @param model * @return */ - @Authority(role = Role.USER) + //@Authority(role = Role.USER) public Object matchV2Action(AddesssMatchModel model) { RestfulModelAndView mav = new RestfulModelAndView(); String result = addressMatchProxyService.parser(convertModelToMap(model)); @@ -526,47 +526,51 @@ public class AddressController { } else if (jsonResult != null) { // 判断是否为自然村 boolean matchedVillage = false; - if (jsonResult.get("searchAddress") != null) { - JSONObject searchAddress = (JSONObject) jsonResult.get("searchAddress"); - String city = searchAddress.getString("city"); - String region = searchAddress.getString("district"); - String name = searchAddress.getString("keywords"); - String township = searchAddress.getString("township"); - if (StringUtils.isNotBlank(city) && StringUtils.isNotBlank(region) && StringUtils.isNotBlank(name)) { - List villages = findVillage(city, region, name); - if (CollectionUtils.isNotEmpty(villages)) { - if (villages.size() == 1 || StringUtils.isBlank(township)) { - JSONObject village = villages.get(0).getJSONObject("_source"); - jsonResult.put("id", villages.get(0).getString("_id")); - jsonResult.put("province", village.getString("province")); - jsonResult.put("city", village.getString("city")); - jsonResult.put("district", village.getString("region")); - jsonResult.put("township", village.getString("township")); - jsonResult.put("name", village.getString("name")); - jsonResult.put("lng", village.getJSONObject("location").getDouble("lon")); - jsonResult.put("lat", village.getJSONObject("location").getDouble("lat")); - jsonResult.put("propertyType", "自然村"); - matchedVillage = true; - } else { - for (JSONObject source : villages) { - JSONObject village = source.getJSONObject("_source"); - if (village.getString("township").contains(township) || township.contains(village.getString("township"))) { - jsonResult.put("id", source.getString("_id")); - jsonResult.put("province", village.getString("province")); - jsonResult.put("city", village.getString("city")); - jsonResult.put("district", village.getString("region")); - jsonResult.put("township", village.getString("township")); - jsonResult.put("name", village.getString("name")); - jsonResult.put("lng", village.getJSONObject("location").getDouble("lon")); - jsonResult.put("lat", village.getJSONObject("location").getDouble("lat")); - jsonResult.put("propertyType", "自然村"); - matchedVillage = true; - break; + try { + if (jsonResult.get("searchAddress") != null) { + JSONObject searchAddress = (JSONObject) jsonResult.get("searchAddress"); + String city = searchAddress.containsKey("city") ? searchAddress.getString("city") : null; + String region = searchAddress.containsKey("district") ? searchAddress.getString("district") : null; + String name = searchAddress.containsKey("keywords") ? searchAddress.getString("keywords") : null; + String township = searchAddress.containsKey("township") ? searchAddress.getString("township") : null; + if (StringUtils.isNotBlank(city) && StringUtils.isNotBlank(region) && StringUtils.isNotBlank(name)) { + List villages = findVillage(city, region, name); + if (CollectionUtils.isNotEmpty(villages)) { + if (villages.size() == 1 || StringUtils.isBlank(township)) { + JSONObject village = villages.get(0).getJSONObject("_source"); + jsonResult.put("id", villages.get(0).getString("_id")); + jsonResult.put("province", village.getString("province")); + jsonResult.put("city", village.getString("city")); + jsonResult.put("district", village.getString("region")); + jsonResult.put("township", village.getString("township")); + jsonResult.put("name", village.getString("name")); + jsonResult.put("lng", village.getJSONObject("location").getDouble("lon")); + jsonResult.put("lat", village.getJSONObject("location").getDouble("lat")); + jsonResult.put("propertyType", "自然村"); + matchedVillage = true; + } else { + for (JSONObject source : villages) { + JSONObject village = source.getJSONObject("_source"); + if (village.getString("township").contains(township) || township.contains(village.getString("township"))) { + jsonResult.put("id", source.getString("_id")); + jsonResult.put("province", village.getString("province")); + jsonResult.put("city", village.getString("city")); + jsonResult.put("district", village.getString("region")); + jsonResult.put("township", village.getString("township")); + jsonResult.put("name", village.getString("name")); + jsonResult.put("lng", village.getJSONObject("location").getDouble("lon")); + jsonResult.put("lat", village.getJSONObject("location").getDouble("lat")); + jsonResult.put("propertyType", "自然村"); + matchedVillage = true; + break; + } } } } } } + } catch(Exception ex) { + ex.printStackTrace(); } if (matchedVillage == false) { jsonResult.put("matchResult", new MatchDistrictResult()); From b7816ccf726ea64aa7794d8b8b41f01991fb6872 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Fri, 17 Feb 2023 13:24:33 +0800 Subject: [PATCH 17/30] =?UTF-8?q?=E5=8C=B9=E9=85=8D=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=E8=87=AA=E7=84=B6=E6=9D=91=E5=8C=B9=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/AddressController.java | 76 ++++++++++--------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index 0167259de..27942a8b1 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -526,47 +526,51 @@ public class AddressController { } else if (jsonResult != null) { // 判断是否为自然村 boolean matchedVillage = false; - if (jsonResult.get("searchAddress") != null) { - JSONObject searchAddress = (JSONObject) jsonResult.get("searchAddress"); - String city = searchAddress.getString("city"); - String region = searchAddress.getString("district"); - String name = searchAddress.getString("keywords"); - String township = searchAddress.getString("township"); - if (StringUtils.isNotBlank(city) && StringUtils.isNotBlank(region) && StringUtils.isNotBlank(name)) { - List villages = findVillage(city, region, name); - if (CollectionUtils.isNotEmpty(villages)) { - if (villages.size() == 1 || StringUtils.isBlank(township)) { - JSONObject village = villages.get(0).getJSONObject("_source"); - jsonResult.put("id", villages.get(0).getString("_id")); - jsonResult.put("province", village.getString("province")); - jsonResult.put("city", village.getString("city")); - jsonResult.put("district", village.getString("region")); - jsonResult.put("township", village.getString("township")); - jsonResult.put("name", village.getString("name")); - jsonResult.put("lng", village.getJSONObject("location").getDouble("lon")); - jsonResult.put("lat", village.getJSONObject("location").getDouble("lat")); - jsonResult.put("propertyType", "自然村"); - matchedVillage = true; - } else { - for (JSONObject source : villages) { - JSONObject village = source.getJSONObject("_source"); - if (village.getString("township").contains(township) || township.contains(village.getString("township"))) { - jsonResult.put("id", source.getString("_id")); - jsonResult.put("province", village.getString("province")); - jsonResult.put("city", village.getString("city")); - jsonResult.put("district", village.getString("region")); - jsonResult.put("township", village.getString("township")); - jsonResult.put("name", village.getString("name")); - jsonResult.put("lng", village.getJSONObject("location").getDouble("lon")); - jsonResult.put("lat", village.getJSONObject("location").getDouble("lat")); - jsonResult.put("propertyType", "自然村"); - matchedVillage = true; - break; + try { + if (jsonResult.get("searchAddress") != null) { + JSONObject searchAddress = (JSONObject) jsonResult.get("searchAddress"); + String city = searchAddress.containsKey("city") ? searchAddress.getString("city") : null; + String region = searchAddress.containsKey("district") ? searchAddress.getString("district") : null; + String name = searchAddress.containsKey("keywords") ? searchAddress.getString("keywords") : null; + String township = searchAddress.containsKey("township") ? searchAddress.getString("township") : null; + if (StringUtils.isNotBlank(city) && StringUtils.isNotBlank(region) && StringUtils.isNotBlank(name)) { + List villages = findVillage(city, region, name); + if (CollectionUtils.isNotEmpty(villages)) { + if (villages.size() == 1 || StringUtils.isBlank(township)) { + JSONObject village = villages.get(0).getJSONObject("_source"); + jsonResult.put("id", villages.get(0).getString("_id")); + jsonResult.put("province", village.getString("province")); + jsonResult.put("city", village.getString("city")); + jsonResult.put("district", village.getString("region")); + jsonResult.put("township", village.getString("township")); + jsonResult.put("name", village.getString("name")); + jsonResult.put("lng", village.getJSONObject("location").getDouble("lon")); + jsonResult.put("lat", village.getJSONObject("location").getDouble("lat")); + jsonResult.put("propertyType", "自然村"); + matchedVillage = true; + } else { + for (JSONObject source : villages) { + JSONObject village = source.getJSONObject("_source"); + if (village.getString("township").contains(township) || township.contains(village.getString("township"))) { + jsonResult.put("id", source.getString("_id")); + jsonResult.put("province", village.getString("province")); + jsonResult.put("city", village.getString("city")); + jsonResult.put("district", village.getString("region")); + jsonResult.put("township", village.getString("township")); + jsonResult.put("name", village.getString("name")); + jsonResult.put("lng", village.getJSONObject("location").getDouble("lon")); + jsonResult.put("lat", village.getJSONObject("location").getDouble("lat")); + jsonResult.put("propertyType", "自然村"); + matchedVillage = true; + break; + } } } } } } + } catch(Exception ex) { + ex.printStackTrace(); } if (matchedVillage == false) { jsonResult.put("matchResult", new MatchDistrictResult()); From 795352405a1792f30b26b98b093c44428ae53e08 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Fri, 17 Feb 2023 13:29:43 +0800 Subject: [PATCH 18/30] =?UTF-8?q?=E5=8C=B9=E9=85=8D=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=E8=87=AA=E7=84=B6=E6=9D=91=E5=8C=B9=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/fangjia/open/controllers/AddressController.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index 3902aaa98..27942a8b1 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -87,7 +87,7 @@ import com.fangjia.service.open.WaitCrawlKeywordsService; import com.fangjia.service.sql.FjtEvaluationService; import com.fangjia.service.web.AddressService; -//@Authority(role = Role.USER) +@Authority(role = Role.USER) public class AddressController { private AddressService addressService; @@ -492,7 +492,7 @@ public class AddressController { * @param model * @return */ - //@Authority(role = Role.USER) + @Authority(role = Role.USER) public Object matchV2Action(AddesssMatchModel model) { RestfulModelAndView mav = new RestfulModelAndView(); String result = addressMatchProxyService.parser(convertModelToMap(model)); From 1ccce2f38ae0ba8188a63686d21d5842ce8691c2 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 17 Feb 2023 14:06:44 +0800 Subject: [PATCH 19/30] update --- .../BigdataCommbuildtypeBatch.groovy | 43 +- .../BigdataCommunityBatch.groovy | 26 +- .../ccbevaluation/BigdataDealBatch.groovy | 74 +- .../ccbevaluation/BigdataForsaleBatch.groovy | 28 +- .../ccbevaluation/CcbFtpTestUtils.groovy | 2 +- .../DealAndThirdPriceDataProcess.groovy | 8 +- .../ExportOracleAllDataTest.groovy | 5 +- .../ExportOracleEvaluationData.groovy | 3 +- .../ccbevaluation/PgsqlCommUtils.groovy | 20 + .../fangjia/script/ccbevaluation/Test.groovy | 101 ++- .../script/ccbevaluation/TradeStatLog.groovy | 12 +- .../pgsql/ExportPgsqlEvaluationResult.groovy | 39 +- .../pgsql/ExportPgsqlRonghe.groovy | 604 +++++++++++++++ .../script/chongqing/HailanTest.groovy | 703 ++++++++++++++++++ .../script/chongqing/MingShenPocStat.groovy | 328 ++++++++ .../script/chongqing/MingShenPocTest.groovy | 459 ++++++++++++ .../script/chongqing/ThreeDESUtils.groovy | 105 +++ .../fuzhou/DistrictSellStatBatch.groovy | 2 + .../com/fangjia/script/fuzhou/Test1.groovy | 2 +- .../fangjia/script/gj/AddAvgpriceBatch.groovy | 1 + .../script/gj/ShLongShangHangLocation.groovy | 52 ++ .../gj/es/FangjiaCorestieLogAnaly.groovy | 42 ++ .../gj/paimai/BeiKeCrawlByChrome.groovy | 4 +- .../gj/paimai/LianjiaCrawlByChrome.groovy | 2 +- .../script/wenyan/temp/DongGuanBank.groovy | 6 +- 25 files changed, 2561 insertions(+), 110 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocStat.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/ThreeDESUtils.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ShLongShangHangLocation.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/FangjiaCorestieLogAnaly.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy index 066318e29..90803d4ca 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy @@ -7,6 +7,7 @@ import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.ccb.CcbCodeUtils import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.util.DateUtils +import groovy.sql.Sql import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService @@ -22,6 +23,9 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified", + "postgres", "postgres", "org.postgresql.Driver") + static main(args) { runBatch(new BigdataCommbuildtypeBatch(), args); } @@ -29,14 +33,14 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ @Override public void process(Object args) { def citys = CcbCommonUtils.getRunningCitys(args) - citys = citys.unique() def rets = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() for(def city : citys){ // def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) // def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","build_type", "lng","c_date", "m_date"]) - def distIds = CcbCommonUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city)); +// def distIds = CcbCommonUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city)); + def distIds = PgsqlCommUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city)) def rows = createDatas(cityCodeDatas.get(city), distIds) rets.addAll(rows) println city +" "+rows.size() @@ -50,7 +54,9 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ def headers = []//["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"] FileUtils.batchCreateFile(rets, fileName,headers, "utf-8", false); - println "txt 生成" + println "txt 生成->导Oracle" + + saveToPgSql(rets) } @@ -103,4 +109,35 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ } return rets; } + + + def saveToPgSql(rows){ + def sqls = [] + for(def r : rows) { + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ + r[i] = null + } + } + r << null + r << '2023-02-28' + r << null + r << '2999-12-31' + r << null + r << null + r << null + } + + def sql = "insert into base.t0831_bigdata_commbuildtype_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + println "start: "+new Date() + def index =0 + sqlInstance.withBatch(20000, sql){ ps-> + for(def r : rows) { + index++ + println index + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14]) + } + } + println "end: "+new Date() + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 1580dda9b..123ada0cf 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -41,7 +41,7 @@ class BigdataCommunityBatch extends BaseBatch{ @Override public void process(Object args) { def citys = CcbCommonUtils.getRunningCitys(args) -// citys = ["文昌"] + citys = ["上海"] def rows = [] def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途", @@ -247,8 +247,7 @@ class BigdataCommunityBatch extends BaseBatch{ println "start: "+new Date() def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" -// def sql = "insert into base.t0831_bigdata_community_gj_h values " - def index = 0 + sqlInstance.withBatch(20000, sql){ ps-> for(def r : rows) { ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], @@ -256,26 +255,7 @@ class BigdataCommunityBatch extends BaseBatch{ ps.toString() } } - for(def r : rows){ - index ++ - if(index == 1){ - sql = sql + "("+r[0].toString()+","+r[1].toString()+","+r[2].toString()+")" - }else{ - sql = sql + ",("+")" - } - } -// sqlInstance.withBatch(50000, sql){ ps-> -// def index = 0 -// for(def r : rows){ -// index++ -// for(def i =0 ;i + count = it.count + }); + return count; + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy index 81958589e..00ed6bf6e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy @@ -7,7 +7,7 @@ import org.apache.commons.codec.digest.DigestUtils class CcbFtpTestUtils { - static def ftpServer = "192.168.0.86"//45 + static def ftpServer = "192.168.0.45"//45 static def ftpPort = 17495 static def ftpUser = "ccb" static def ftpPassword = "ccb!@#" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index 6b28570f1..3123dc829 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -25,9 +25,9 @@ class DealPriceDataProcess { def task = new DealPriceDataProcess() // task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据 // task.addLianJiaPrice(); -// task.deleteThirdPartyData() + task.deleteThirdPartyData() // task.delDealByPe() - task.delDealData() +// task.delDealData() } def delDealByPe(){ @@ -372,12 +372,12 @@ class DealPriceDataProcess { def deleteThirdPartyData(){ def citys = CcbCommonUtils.citys_all - def month = 202211 + def month = 202212 def rows = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def set = new HashSet() for(def city : citys) { - def rets = findPeGteThirdParty(city, 202212) + def rets = findPeGteThirdParty(city, 202301) def total = 0 println city +" "+rets.size() if(rets){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy index 39ef500ac..4df84ea86 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy @@ -13,6 +13,7 @@ import org.apache.commons.codec.digest.DigestUtils import java.sql.ResultSet +//估值 新版 导出所有数据 class ExportOracleAllDataTest { def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", @@ -20,7 +21,7 @@ class ExportOracleAllDataTest { def cityCodeMap = CcbCommonUtils.findCityCodeValue() def citys = null - def curDate = "20221228"//DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def curDate = "20220128"//DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; static main(args) { def task = new ExportOracleAllDataTest() @@ -38,7 +39,7 @@ class ExportOracleAllDataTest { def run(args) { citys = CcbCommonUtils.getRunningCitys(args)//["潍坊","济宁","威海","临沂","绵阳","陇南","塔城","阿勒泰","迪庆州","日喀则"] //CcbCommonUtils.getRunningCitys(args) - def curMonth = "202212"//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); + def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") def new_dists = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy index 419f170b0..513e7025c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy @@ -31,7 +31,7 @@ class ExportOracleEvaluationData { exportCityPrice(curMonth) exportRegionPrice(curMonth) - exportCommunityBasePriceGj(curMonth)//已合下面的合并 + // exportCommunityBasePriceGj(curMonth)//已合下面的合并 exportCommunityBasePrice(curMonth) exportCommunityPdescribeGj(curMonth) @@ -248,6 +248,7 @@ class ExportOracleEvaluationData { def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy new file mode 100644 index 000000000..f2a564cb7 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy @@ -0,0 +1,20 @@ +package com.fangjia.script.ccbevaluation + +import groovy.sql.Sql + +class PgsqlCommUtils { + + public static def database = "warehouse" + + public static def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/"+database+"?stringtype=unspecified", + "postgres", "postgres", "org.postgresql.Driver") + + public static def findAllDistrictIdsByOracle(def cityid){ + def sql = "select gujia_comm_id from base.t0831_bigdata_community_gj_h where urbn_id='${cityid}' and gujia_bds_bds_st=0"; + def set = new HashSet() + sqlInstance.eachRow(sql,{r-> + set << r.gujia_comm_id + }); + return set + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 14a238118..8aca6bac6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -45,8 +45,91 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl) +def citys = ["山南","黄南州","甘南","克孜勒苏","嘉峪关","海北州","阿拉善盟","怒江州","和田","日喀则","昌都","海南州","玉树州","吐鲁番","阿勒泰","三沙","塔城","甘孜州","果洛州","林芝","那曲","阿坝州","神农架"] +def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +ExcelUtils.multiThreadingModify("D:\\数据\\小区\\小区地址名单_估家科技V2 (3).xlsx",{d-> + if(d['J']){ + def cityId = cityCodeDataMap.get(d['H']) + def map = findCcbPriceCredit(sqlInstance236, cityId, d['J']) + println d['H'] +" "+d['J']+ map.toString() + if(map){ + d['Q'] = map.credit + d['R'] = map.month + d['U'] = map.price + } +// def thirdMap = findThirdPriceBySource(sqlInstance236, cityId, d['J']) +// if(thirdMap){ +// d['S'] = thirdMap.price +// d['T'] = thirdMap.month +// } + } + +}) + +def findCcbPriceCredit(sqlInstance, cityid, commid){ + def sql = "select * from (select CONFIDENCE,year_month,avg_price from community_base_price_gj where cityid='${cityid}' and commid='${commid}' order by year_month desc) WHERE ROWNUM <= 1" + def map = [:] + sqlInstance.eachRow(sql,{r-> + map.month = r.year_month + map.credit = r.CONFIDENCE + map.price = r.avg_price + }); + return map +} + +def findThirdPriceBySource(sqlInstance, cityid, commid){ + def sql = "select * from (select data_source, price_date, community_price,commid from bigdata_third_party_price_gj where cityid='${cityid}' and commid='${commid}' and data_source in ('贝壳','链家') \n" + + "order by price_date desc) WHERE ROWNUM <= 1" + def map = [:] + sqlInstance.eachRow(sql,{r-> + map.month = r.price_date + map.price = r.community_price + }); + return map +} + + + + + +for(def c : citys){ + def cityCode = cityCodeDataMap.get(c) + println c +" "+cityCode + def priceDists = findAllPriceDist(sqlInstance236,cityCode, 202301) +// + checkNoPriceDist(sqlInstance236, priceDists, cityCode) +} + +def checkNoPriceDist(sqlInstance236,priceDists, cityid){ + def commids = [] + def sql = "select commid from bigdata_community_gj where cityid='${cityid}' and status=0"; + sqlInstance236.eachRow(sql, { r -> + commids << r.commid + }) + + for(def commid : commids){ + if(!priceDists.contains(commid)){ + updateDistStatus(sqlInstance236, commid) + } + } + return commids +} + + +def updateDistStatus(sqlInstance236, commid){ + def sql = "update bigdata_community_gj set status=1 where commid='${commid}'"; + sqlInstance236.executeUpdate(sql) +} +def findAllPriceDist(sqlInstance236, cityid, yearMonth){ + def commids = [] + def sql = "select commid from community_base_price_gj where cityid='${cityid}' and year_month=${yearMonth} "; + sqlInstance236.eachRow(sql, { r -> + commids << r.commid + }) + return commids +} //def citys = ['苏州','杭州','宁波','温州','嘉兴','湖州','绍兴','金华','衢州','舟山','台州','丽水'] // //def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() @@ -1087,15 +1170,15 @@ def findThirdCommCount(cityid, sqlInstance){ //} -def fileName = "D:\\数据\\建行\\房产估值\\指导价\\国信达价格-杭州.xlsx" -ExcelUtils.multiThreadingModify(fileName, { d -> - def dist = CommonDatautils.findDistrictById(d['A']) - if(dist){ - d['C'] = dist.city - d['D'] = dist.region - d['E'] = dist.name - } -}) +//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\国信达价格-杭州.xlsx" +//ExcelUtils.multiThreadingModify(fileName, { d -> +// def dist = CommonDatautils.findDistrictById(d['A']) +// if(dist){ +// d['C'] = dist.city +// d['D'] = dist.region +// d['E'] = dist.name +// } +//}) //def fileName = "D:\\数据\\建行\\房产估值\\指导价\\0922入库数据.xlsx" //def datas = ExcelUtils.read(fileName, 0, 1) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index e87103a90..92fb500fc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -40,7 +40,7 @@ class TradeStatLog { static main(args) { def task = new TradeStatLog() -// task.run()//统计最近3个月抓取的成交数据量 + task.run()//统计最近3个月抓取的成交数据量 // task.exportTrade()//导出未入库成交 // task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) @@ -92,13 +92,13 @@ class TradeStatLog { def run(){ def citys = CcbCommonUtils.citys_all; def rets = [] - rets << ["城市","9月总量","10月总量","11月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] + rets << ["城市","10月总量","11月总量","12月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"] //def startDate = "2021-05-01" - def startDate = "2022-09-01" - def date = "2022-10-01" - def endDate = "2022-11-01" - def newDate = "2022-12-01" + def startDate = "2022-10-01" + def date = "2022-11-01" + def endDate = "2022-12-01" + def newDate = "2023-01-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy index 1508d0b9f..24254df78 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy @@ -1,8 +1,10 @@ package com.fangjia.script.ccbevaluation.pgsql import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.ccbevaluation.CcbFtpTestUtils import com.fangjia.script.ccbevaluation.CcbFtpUtils import com.fangjia.script.ccbevaluation.ronghe.SSHHelper +import com.fangjia.script.ccbevaluation.ronghe.SSHHelperTest import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.util.DateUtils @@ -43,13 +45,11 @@ class ExportPgsqlEvaluationResult { // exportCommunityPdescribeGj(curMonth) // exportCommCaseDetailGj(curMonth) // - def curDate = curMonth.toString()+"28" -// 生成MD5文件 - exportMd5() - SSHHelper jschCommand = new SSHHelper(); - System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}")); -// ok文件 - exportOkFile() + exportInOk() + + SSHHelperTest jschCommand = new SSHHelperTest(); + System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}")); + } def exportAvmEvaluationGJNB(yearMonth){ @@ -363,20 +363,31 @@ class ExportPgsqlEvaluationResult { } - def exportOkFile(){ - def dataCodeFileName = linux_prefix + "${curDate}.ok" - if(CommonDatautils.isWindow()){ - dataCodeFileName = win_prefix + "${curDate}.ok" + def exportInOk(){ + Vector files = CcbFtpTestUtils.listFileNames(null, curDate) + Iterator fileVec = files.iterator() + def fileRows = [] + while(fileVec.hasNext()){ + ChannelSftp.LsEntry file = fileVec.next() + + if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){ + fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(null, file.getFilename(), curDate) + } } - createFileToFtp("ok txt", [], dataCodeFileName, curDate, false) + def fileName = linux_prefix + "/${curDate}.ok" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "${curDate}.ok" + } + + createFileToFtp("ok txt",fileRows, fileName, curDate, false) } def createFileToFtp(tableName,allRows, fileName, curDate, isAppend){ FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", isAppend); println tableName+" 数据文件输出完成......" - CcbFtpUtils.ftpUpload(null, fileName, curDate) -// println tableName+" 上传Ftp完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println tableName+" 上传Ftp完成......" } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlRonghe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlRonghe.groovy index 8d3d01341..5a4068b7c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlRonghe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlRonghe.groovy @@ -1,14 +1,618 @@ package com.fangjia.script.ccbevaluation.pgsql +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.ccbevaluation.CcbFtpTestUtils +import com.fangjia.script.ccbevaluation.CcbFtpUtils +import com.fangjia.script.ccbevaluation.ronghe.SSHHelperTest +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.jcraft.jsch.ChannelSftp +import groovy.sql.Sql + class ExportPgsqlRonghe { + def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/postgres", + "postgres", "postgres", "org.postgresql.Driver") + + def citys = [] + + def database = "base." + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + + def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") + def linux_prefix = "/ccb_ronghe/${curDate}/" + def win_prefix = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\" + static main(args) { def task = new ExportPgsqlRonghe() task.run(args) } def run(args) { + def new_dists = [] + citys = CcbCommonUtils.getRunningCitys(args) + //按天导(每天一个文件夹) + exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date() + 1, -1), "yyyy-MM-dd"), new_dists) + if (curDate.toString().endsWith("10")) { + + exportEvaluateRate() + exportBigdataBuildprojFjw() + exportBigdataCommlayoutpicFjw() + + + exportBigdataCommunityGJRH()// + exportBigdataBuildingFJW()// + exportBigdataCommfeeGj()// + + exportBigdataCommbuildstruGJ()// + exportBigdataCommhousetypeGJ()// + exportBigdataCommbuildyearGj()// + exportBigdataCommhouseuseGJ()// + exportBigdataPositionGJ()// + exportBigdataDistrictFJW()// + exportBigdataAliasFJW()// + + + //本12.10上传10,11月所有数据,23年1月需改为所有城市按日期同步一期数据 + def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() +// exportBigdataCityPrice(queryDate) +// exportBigdataDistPrice(queryDate) +// exportBigdataCommPrice(queryDate) + + exportInOk() + +// } + +// SSHHelperTest jschCommand = new SSHHelperTest(); +// System.out.println(jschCommand.executeCommand("sh /home/ccb_ronghe/output/rh_package.sh ${curDate}")); + } + } + + + def exportBigdataDealFJW2(curDate, dealdate, new_dists){ + + def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd") + def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd") + + def fileName = "FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + } + + def allRows = [] + + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + if(curDate.toString().endsWith("10")){ + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " + + "from "+database+"t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>=to_date('${dealDate}','yyyy-MM-dd') and gujia_admndiv_id not in ${newstr}" + def rets1 = dealData(sql1) + + def sql2 = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " + + "from "+database+"t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr}" + def rets2 = dealData(sql2) + allRows.addAll(rets1) + allRows.addAll(rets2) + }else{ + def sql = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " + + "from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>=to_date('${dealDate}','yyyy-MM-dd')"; + allRows.addAll(dealData(sql)) + } + } + } + createFileToFtp("BIGDATA_DEAL_FJW", allRows, fileName, curDate, false) + } + + + + def dealData(def sql){ + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.urbn_id + data << r.urbn_id + data << r.mdl_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_str + data << r.gujia_hsno + data << r.gujia_comm_id + data << r.comm_nm + data << r.gujia_bld_id + data << r.bld_id + data << r.gujia_bld_nm + data << r.gujia_unit_id + data << r.unit_id + data << r.gujia_unit_nm + data << r.gujia_acc_id + data << r.gujia_dtl_adr + data << r.gujia_bld_yr_rc + data << r.gujia_own_val + data << r.gujia_own_yrlmt_val + data << r.gujia_prd_val + data << r.gujia_wbt_floor + data << r.gujia_lyr_tp_val + data << r.gujia_tot_floor_num + if(r.gujia_bld_tp_val){ + def arrs = r.gujia_bld_tp_val.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.gujia_bld_stc + data << r.gujiaconstructionarea + data << r.gujia_set_innr_area + data << r.gujia_acc_tp_val + data << r.gujia_room + data << r.gujia_dept + data << r.gujia_ktch_num + data << r.gujia_toliet + data << r.gujia_hs_use + data << r.gujia_indr_stc_val + data << r.gujia_twrd_val + data << r.gujia_dcrt_sttn + data << r.gujia_ases_bt_ln_num + data << r.gujia_trdtckt_prc + data << r.gujia_mdl_tot_prc + data << r.gujia_dcl_tot_prc + data << r.gujia_ases_tot_prc + data << r.gujiasigntonlinectrid + data << r.txn_dt + data << r.gujia_txn_tp_val + data << r.gujia_sell_pymtmod + data << r.gujia_mdl_cyc + data << r.gujiawthrext_lift_ind + data << r.gujiagvlndfargrdnarea + data << r.gujia_gv_lu_tbl_area + data << r.gujia_gv_sun_tbl_area + data << r.gujiagvbaywindow_area + data << r.gujiagvarea_tower_flr + data << r.gujia_gv_grg_area + data << r.gujiagvundgdroom_area + data << r.gujia_gv_othr_area + data << r.gujia_hs_src_dtsrc + data << r.intr_co_id + data << r.intr_co_nm + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + data << r.gujia_mkdam + data << r.gujia_data_tm + allRows << data + }); + return allRows + } + + def exportEvaluateRate(){ + def sql = "select gujia_ceft_id,gujia_ceft_tp,gujia_ceft,gujia_seq, gujia_urbn,rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from t0831_evaluate_rate_fjw_w"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.gujia_ceft_id + data << r.gujia_ceft_tp + data << r.gujia_ceft + data << r.gujia_seq + data << r.gujia_urbn + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + def fileName = linux_prefix + "FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" + } + + createFileToFtp("EVALUATE_RATE_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataBuildprojFjw(){ + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_BUILDPROJ_FJW", [], fileName, curDate, false) + } + + def exportBigdataCommlayoutpicFjw(){ + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMLAYOUTPIC_FJW", [], fileName, curDate, false) + } + + def exportBigdataCommunityGJRH(){ + + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" + } + def allRows = [] + for(def city : citys){ + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_comm_id,comm_nm,gujia_comm_fmnm,gujia_admndiv_id,gujia_str,gujia_hsno,gujia_dtl_adr,gujiaocpoflandtotarea,gujia_bld_tot_area,gujia_comm_tp_val,gujiawthr_new_pan_ind,gujia_grnrate,gujia_vol_rate,gujia_park_num,gujia_comm_intd,rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujiaprhsld_hspty_prc,gujiacomm_tot_rsd_num,gujiacommrntfee_avprc,gujia_comm_avprc from "+database+"t0831_bigdata_community_fjw_w where urbn_id='${cityCode}'"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_comm_id + data << r.comm_nm + data << r.gujia_comm_fmnm + data << r.gujia_admndiv_id + data << r.gujia_str + data << r.gujia_hsno + data << r.gujia_dtl_adr + data << r.gujiaocpoflandtotarea + data << r.gujia_bld_tot_area + data << r.gujia_comm_tp_val + data << r.gujiawthr_new_pan_ind + data << r.gujia_grnrate + data << r.gujia_vol_rate + data << r.gujia_park_num + data << r.gujia_comm_intd + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + createFileToFtp("BIGDATA_COMMUNITY_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataCommbuildstruGJ(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id, gujia_comm_id, gujia_bld_stc, rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+"t0831_bigdata_commbuildstru_fjw_w where urbn_id='${cityCode}'"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_comm_id + data << r.gujia_bld_stc + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMBUILDSTRU_FJW", allRows, fileName, curDate, false) } + def exportBigdataCommfeeGj(){ + + def allRows = [] + for(def city : citys){ + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_comm_id,co_id,gujia_eps_val,gujia_hs_use,gujia_bld_tp_val,gujia_comm_eps_tp,gujia_eps_cmnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_commfee_fjw_w where urbn_id='${cityCode}'"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_comm_id + data << r.co_id + data << r.gujia_eps_val + data << r.gujia_hs_use + if(r.gujia_bld_tp_val){ + def arrs = r.gujia_bld_tp_val.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.gujia_comm_eps_tp + data << r.gujia_eps_cmnt + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMFEE_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataCommhousetypeGJ(){ + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_comm_id,gujia_comm_hs_tp,rmrk_dsc,to_char(data_crt_tm'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_commhousetype_fjw_w where urbn_id='${cityCode}'"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_comm_id + data << r.gujia_comm_hs_tp + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMHOUSETYPE_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataCommbuildyearGj(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_comm_id,gujia_bld_yr_rc,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_commbuildyear_fjw_w where urbn_id='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_comm_id + data << r.gujia_bld_yr_rc + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMBUILDYEAR_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataCommhouseuseGJ(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_comm_id,gujia_hs_use,gujia_ladder_accnum,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_commhouseuse_fjw_w where urbn_id='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_comm_id + data << r.gujia_hs_use + data << r.gujia_ladder_accnum + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMHOUSEUSE_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataPositionGJ(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_obj_tp,gujia_obj_id,gujia_lo_tp,gujia_sn,lgt,ltt,gujia_hignas_val,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_position_fjw_w where urbn_id='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_obj_tp + data << r.gujia_obj_id + data << r.gujia_lo_tp + data << r.gujia_sn + data << r.lgt + data << r.ltt + data << r.gujia_hignas_val + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_POSITION_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataDistrictFJW(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_admndiv_id,gujia_admndiv_nm,gujia_admndiv_nm_cpa,urbn_id,gujia_natstd_cd,gujia_dspl_seq,gujia_admndiv_cgy,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"t0831_bigdata_district_fjw_w where urbn_id='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_admndiv_nm_cpa + data << r.urbn_id + data << r.gujia_natstd_cd + data << r.gujia_dspl_seq + data << r.gujia_admndiv_cgy + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_DISTRICT_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataAliasFJW(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,rel_id,gujia_comm_id,comm_nm,gujia_pcsst,gujia_vld_dt,gujia_vrsn_dsc,gujia_wthr_prim_nm,gujia_dtsrc,gujia_dsc,gujia_als_lvl,gujia_adr_tp_ecd,gujia_wthr_adt_ind,gujia_wthr_dspl_ind from "+database+"t0831_address_alias_fjw_w where urbn_id='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.rel_id + data << r.gujia_comm_id + data << r.comm_nm + data << r.gujia_pcsst + data << r.gujia_vld_dt + data << r.gujia_vrsn_dsc + data << r.gujia_wthr_prim_nm + data << r.gujia_dtsrc + data << r.gujia_dsc + data << r.gujia_als_lvl + data << r.gujia_adr_tp_ecd + data << r.gujia_wthr_adt_ind + data << r.gujia_wthr_dspl_ind + allRows << data + }); + } + def fileName = linux_prefix + "FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" + } + createFileToFtp("ADDRESS_ALIAS_FJW", allRows, fileName, curDate, false) + } + + def exportBigdataBuildingFJW(){ + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" + } + def index = 0 + for(def city : citys) { + def allRows = [] + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_bld_id,bld_id,gujia_bld_nm,gujia_bld_als_dsc,gujia_admndiv_id,gujia_comm_id,gujia_bld_yr_rc,gujia_bld_tp_val,gujia_bld_stc,floor_num,gujia_unit_num,gujia_accnum_val,gujiawthrext_lift_ind,gujia_lift_num,gujiajiladderjiactval,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_vld_ind from "+database+"t0831_bigdata_building_fjw_w where urbn_id='${cityCode}'";// + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_bld_id + data << r.bld_id + data << r.gujia_bld_nm + data << r.gujia_bld_als_dsc + data << r.gujia_admndiv_id + data << r.gujia_comm_id + data << r.gujia_bld_yr_rc + if(r.gujia_bld_tp_val){ + def arrs = r.gujia_bld_tp_val.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.gujia_bld_stc + data << r.floor_num + data << r.gujia_unit_num + data << r.gujia_accnum_val + data << r.gujiawthrext_lift_ind + data << r.gujia_lift_num + data << r.gujiajiladderjiactval + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + data << r.gujia_vld_ind + allRows << data + }); + + if(allRows && allRows.size()>0){ + index = index + 1 + if(index == 1){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + } + } + } + + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_BUILDING_FJW 上传Ftp完成......" + } + + def exportBigdataCityPrice(queryDate){ + def fileName = linux_prefix + "FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat" + } + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + // def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";// + def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from t0831_city_price_gj_h where CITYID='${cityCode}'";// + + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.YMID + data << r.SALEPRICE?r.SALEPRICE:'' + data << r.RENTPRICE?r.RENTPRICE:'' + if(r.SALECOUNT>0) { + data << r.SALECOUNT + }else{ + data << '' + } + if(r.RENTCOUNT>0){ + data << r.RENTCOUNT + }else{ + data << '' + } + data << r.UPDATETIME + data << r.HOUSEUSE + data << r.SALEBLDGAREA?r.SALEBLDGAREA:'' + data << r.RENTBLDGAREA?r.RENTBLDGAREA:'' + data << r.SALETOTALPRICE?r.SALETOTALPRICE:'' + data << r.RENTTOTALPRICE?r.RENTTOTALPRICE:'' + data << r.DATATYPE + data << r.SDATERANGE + data << r.RDATERANGE + data << r.SOURCECODE + allRows << data + }); + } + + createFileToFtp("BIGDATA_CITYPRICE", allRows, fileName, curDate, false) + } + + def exportInOk(){ + Vector files = CcbFtpTestUtils.listFileNames("rh", curDate) + Iterator fileVec = files.iterator() + def fileRows = [] + while(fileVec.hasNext()){ + ChannelSftp.LsEntry file = fileVec.next() + + if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){ + fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength("rh", file.getFilename(), curDate) + } + } + def fileName = linux_prefix+"${curDate}.ok" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "${curDate}.ok" + } + createFileToFtp("融合统计文件长度ok文件", fileRows, fileName, curDate, false) + } + + def createFileToFtp(tableName,allRows, fileName, curDate, isAppend){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", isAppend); + println tableName+" 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload("rh", fileName, curDate) + println tableName+" 上传Ftp完成......" + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy new file mode 100644 index 000000000..2b163d322 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy @@ -0,0 +1,703 @@ +package com.fangjia.script.chongqing + +import com.fangjia.core.RegionData +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import net.sf.json.JSONObject +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Matcher +import java.util.regex.Pattern + +class HailanTest { + + static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"] + private static final HashMap> FITMENTS_TYPES_MAPS = new HashMap>() + static{ + FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"]) + FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"]) + FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"]) + FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"]) + } + + static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) + def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + + static main(args) { + def task = new HailanTest(); + task.run(); + } + + def run(){ + def inputfile = "D:\\数据\\海南\\统计.xlsx" +// match(inputfile) +// addPrice(inputfile) +// addCommId("D:\\数据\\海南\\匹配上的合并数据.xlsx") +// addHouse(inputfile) +// stat(inputfile) + hailanStat(inputfile) + } + + def addHouse(inputfile){ + + def datas = [] + datas << ["类型(住宅/别墅)","行政区划","详细地址","房号","小区名称","建筑面积","所在楼层","详细地址","小区名","交易日居住用房价格指数","评估日居住用房价格指数","是否有电梯","所在楼层","总楼层","内部装修","平面布置","朝向", + "景观","建筑结构","建筑面积(平方米)","房屋可使用年限","房屋已使用年限","评估价(单价)","挂牌价(单价)","成交价(单价)","小区/楼盘均价(单价)"] + def index = 0 + ExcelUtils.multiThreadingModify(inputfile, { row -> + if(row['L']){ + index ++ + def dist = CommonDatautils.findDistrict(row['H'], row['I'], row['J'], "district") + def rets = findHouses(row['H'], row['I'], "sell", 20200101,20230113, row['J'], null, row['k']) + println index +" "+ row['C']+" "+ rets.size() + if(rets){ + def lastRets = [] + for(def r : rets){ + def fjwPrice = row['M'] + if(fjwPrice){ + def precent = java.lang.Math.abs((fjwPrice-r.avg_price)*100/r.avg_price) + if(precent>=10){ + continue; + } + } + if(row['A'] && row['A']=="别墅"){ + def inputAare = row['F'].toString().toDouble() + def minArea = inputAare + 50 + def maxArea = inputAare + 100 + if(r.areamaxArea){ + continue; + } + } + + lastRets << r + if(lastRets.size()>=3){ + break; + } + } + + if(lastRets){ + for(def lr : lastRets){ + def data = [] + data << row['A'] + data << row['B'] + data << row['C'] + data << row['D'] + data << row['E'] + data << row['F'] + data << row['G'] + if(dist){ + data << dist.address + }else{ + data << "" + } + if(dist){ + data << dist.name + }else{ + data << "" + } + + data << "" + data << "" + if(lr.height){ + if(lr.height>6){ + data << "是" + }else{ + data << "否" + } + }else{ + data << "" + } + data << lr.floor + data << lr.height + data << lr.fitment + data << "" + data << lr.direction + data << "" + data << lr.build_type + data << lr.area + data << dist.property_life +// def year = dist.create_year +// if(year){ +// data << getHouseYear(year.toInteger())//已使用年限 +// }else{ +// data << ""//已使用年限 +// } + data << getHouseYear(dist.create_year , lr.build_date, dist.build_date) + + data << getEvalutaionPrice(lr, dist)//评估价(单价) + data << lr.avg_price + data << getDealPrice(dist)//成交单价 + data << row['M'] + datas << data + } + } + } + } + }) + ExcelUtils.write("D:\\数据\\海南\\海南样本数据.xlsx", datas, "样本数据") + } + + def getDealPrice(dist){ + def json = OpenApiClient.searchDealPrice(dist._id.toString(),"20220101", 100) + def prices = [] + if(json && json.code==200){ + def result = json.result + if(result){ + for(def r : result){ + prices << r.avgPrice + } + } + } + def dealAvgPrice = null + if(prices){ + dealAvgPrice = (prices.sum()/prices.size()).toInteger() + } + return dealAvgPrice; + } + + def getHouseYear(create_year , house_build_date, dist_build_date){ + def year = null + if(house_build_date){ + year = getYear(house_build_date) + + }else if(dist_build_date){ + year = getYear(dist_build_date) + }else if(create_year){ + year = create_year + } + if(year){ + year = year.toInteger() + def curYear = new Date().getYear()+1900 + def modYear = curYear - year + return modYear.toInteger().toString() + } + return null + + } + + + def getYear(finishBuildingDate){ + if(finishBuildingDate){ + Pattern pat3 = Pattern.compile("\\d{4}"); + def matcher3 = pat3.matcher(finishBuildingDate); + if(matcher3.find()){ + return matcher3.group() + } + } + return null + } + + + def getEvalutaionPrice(house, dist){ + def params = [:] + params.put("city", dist.city); + params.put("district", dist.region); + params.put("name", dist.name); + params.put("size", String.valueOf(house.area)); + if(house.height) { + params.put("maxFloor", dist.house_height.toString()); + params.put("direction", getDirectionCode(house.direction).toString());//朝向 + if (house.house_floor) { + params.put("floor", house.house_floor.toString()); + } + } + def fitmentCode = null + if(house.fitment){ + fitmentCode = getFitmentCode(house.fitment) + params.put("decoration", fitmentCode.toString());//装修 + } + params.put("scope", "1") + def result = OpenApiClient.evaluateDistrictPrice(params); + + if(result){ + return result.avgPrice + } + return null + } + + + def getDirectionCode(def s){ + def directMap = ["西南":1, "东南":2, "南":3, "西北":4, "东北":5, "东":6, "西":7, "北":8, "东西":9, "南北":10] + return directMap.get(s); + } + def getFitmentCode(def s){ + if(s){ + if(["毛坯"].contains(s)){ + return 1; + }else if(["简单装修","简装"].contains(s)){ + return 2; + }else if(["精装修","精装"].contains(s)){ + return 3; + }else if(["豪华装修","豪装"].contains(s)){ + return 4; + } + } + return null + } + + + + + def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range) + + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + + + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + + def rets = [] + def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"] + def index = new Random().nextInt(5); + + def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder); + + if(houses){ + for (int k=0;k60){ + continue; + } + +// if(!seawater.room){ +// continue; +// } +// if(!seawater.room){ +// seawater.room = null +// } + + if(seawater.room){ + if(seawater.room>=11 || seawater.room<1){ + continue; + } + } + + if(!seawater.hall){ + seawater.hall = null + } + if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){ + continue; + } + if(!seawater.toilet){ + seawater.toilet = null + } + if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){ + continue; + } + + + def floor = parseFloor(seawater.floor_info) + if(seawater.height && floor && floor.isNumber()){ + if(floor.toInteger()>seawater.height){ + floor = null + seawater.height = null + } + + } + + if(!floor){ + seawater.floor = null + } + if(floor && floor.toInteger()<1){ + floor = null + } + + def rowMap = [:]; + + rowMap.houseId = seawater.id +// rowMap.city = CcbCommonUtils.getCcbCity(city) + //rowMap.region =seawater.region + //rowMap.address = seawater.address + rowMap.area = seawater.area + rowMap.address = seawater.address + rowMap.total_price = seawater.total_price + rowMap.avg_price = seawater.avg_price + rowMap.build_type = seawater.build_type + rowMap.build_date = seawater.build_date + rowMap.district_name = seawater.district_name + rowMap.height = seawater.height=="null"?null:seawater.height + if(rowMap.height && rowMap.height<1){ + rowMap.height = null + } + rowMap.floor = floor=="null"?null:floor + rowMap.room = seawater.room=="null"?null:seawater.room + rowMap.hall = seawater.hall=="null"?null:seawater.hall + rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet + + + def direction = parseDirection(seawater.direction) + rowMap.direction = direction=="null"?null:direction + + def fitment = parseFitment(seawater.fitment) + rowMap.fitment = fitment=="null"?null:fitment + + + + if(seawater.p_date instanceof String){ + seawater.p_date = seawater.p_date.toInteger() + } + + rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") + rowMap.c_date = DateUtils.getConverDate(seawater.m_date) + rowMap.m_date = DateUtils.getConverDate(seawater.m_date) + + rets << rowMap + } + } + return rets; + } + + + static def parseFloor(def floorDesc){ + if(floorDesc){ + floorDesc = floorDesc.replaceAll("[\\s ]+", "") + if(floorDesc){ + Pattern pat1 = Pattern.compile("第(\\d{1,2})层"); + Matcher matcher1 = pat1.matcher(floorDesc); + if (matcher1.find()) { + return matcher1.group(1) + } + pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?"); + matcher1 = pat1.matcher(floorDesc); + if (matcher1.find()) { + return matcher1.group(1) + } +// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){ +// return "中层" +// } +// +// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){ +// return "高层" +// } +// +// +// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){ +// return "低层" +// } + if(floorDesc.isNumber()){ + return floorDesc; + } + } + } + return null; + } + + static String parseDirection(String directionDesc) { + if (directionDesc == null) { + return null + } + directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#") + StringBuffer sb = new StringBuffer() + for (int i = 0; i < directionDesc.length(); i++) { + String c = directionDesc.substring(i, i + 1) + if (DIRECTIONS.contains(c)) { + if (directionDesc.substring(i).startsWith(c + "#_#")) { + sb.append(c + " ") + } else { + sb.append(c) + } + } + } + for (String d : DIRECTIONS) { + if (sb.toString().startsWith(d)) { + return d + } + } + for (String d : DIRECTIONS) { + if (sb.toString().contains(d)) { + return d + } + } + return null + + } + + static String parseFitment(String fitmentDesc) { + if (fitmentDesc == null) { + return null + } + fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "") + if (fitmentDesc.equals("非毛坯")) { + return null + } + Iterator>> it = FITMENTS_TYPES_MAPS + .entrySet().iterator() + Map.Entry> entry + while (it.hasNext()) { + entry = it.next() + if (fitmentDesc.contains(entry.getKey())) { + return entry.getKey() + } + for (String fitmentKey : entry.getValue()) { + if (fitmentDesc.contains(fitmentKey)) { + return entry.getKey() + } + } + } + return null + + } + + + + def addCommId(inputfile){ + ExcelUtils.modify(inputfile, { row -> + if(row['H']){ + def dist = CommonDatautils.findDistrict(row['H'], row['I'], row['J'], "district") + if(dist){ + row['L'] = dist._id.toString() + } + } + }) + } + + def addPrice(path){ + ExcelUtils.multiThreadingModify(path, { row -> + def commid = row['L'] + if(commid){ + def dist = CommonDatautils.findDistrictById(commid.trim()) + if(dist){ + def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name) + if(priceObj){ + row['M'] = priceObj.avg_price + } + } + } + }) + } + + + def match(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(row['C']){ + String city = row['H']//RegionData.formatCity(row['B']) + if(city){ + def address = row['C'] +// def region = row['C'] + println "[match] city:"+city+" address:"+address + def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, null, row['E']) + if(jsonObject != null && jsonObject.precise!=-1) { + row['I'] = jsonObject.get("district") + row['J'] = jsonObject.get("name") + row['K'] = jsonObject.get("propertyType") + row['L'] = jsonObject.get("id") + } + }else{ + println row['C'] + } + + } + }) + } + + def stat(path){ + ExcelUtils.modify(path, { row -> + if(row['L'] && !row['H']){ + def city = row['L'] + def region = row['M'] + def dists = null + if(city && region){ + dists = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city","region","name","alias","estate_type2"]) + }else{ + dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias","estate_type2"]) + } + def distLists = [] + def villaLists = [] + for(def d : dists){ + if(d.estate_type2=="别墅"){ + villaLists << d + }else{ + distLists << d + } + } + row['J'] = dists.size() + + def haveHouseList = findHaveHouseDist(distLists) + def haveVillaList = findHaveHouseDist(villaLists) + row['H'] = haveHouseList + row['I'] = haveVillaList +// row['N'] = distLists.size() +// row['O'] = villaLists.size() + } + }) + } + + def hailanStat(path){ + def total = 0 + def distTotal = 0 + def villaTotal = 0 + def haveHouseDistTotal = 0 + def haveHouseVillaTotal = 0 + def buildStructureTotal = 0 +// ExcelUtils.modify(path, { row -> +// if (row['L'] && !row['K']) { + //"海口","三亚","三沙","儋州", + def citys = ["五指山","琼海","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"] + for(def city : citys) { + + def dists = null +// if (city && region) { +// dists = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city", "region", "name", "alias", "estate_type2", "build_structure"]) +// } else { + dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias", "estate_type2", "build_structure"]) +// } + def distLists = [] + def villaLists = [] + for (def d : dists) { + if (d.estate_type2 == "别墅") { + villaLists << d + } else { + distLists << d + } + } + total = total + dists.size() + +// def structureCount = 0 +// for(def d : dists){ +// if(d.build_structure){ +// structureCount ++ +// } +// } +// println structureCount +// row['K'] = structureCount +// buildStructureTotal = buildStructureTotal + structureCount +// + def haveHouseList = findHaveHouseDist(distLists) + def haveVillaList = findHaveHouseDist(villaLists) + haveHouseDistTotal = haveHouseDistTotal + haveHouseList + haveHouseVillaTotal = haveHouseVillaTotal + haveVillaList + distTotal = distTotal + distLists.size() + villaTotal = villaTotal + villaLists.size() +// println c +" "+total+" "+buildStructureTotal +// } + } +// }) + + println total +" "+haveHouseDistTotal+" "+haveHouseVillaTotal+" "+distTotal+" "+villaTotal+" "+buildStructureTotal + + } + + def findHaveHouseDist(districts){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + println "小区量:"+districts.size() + + def rets = new CopyOnWriteArrayList() + + for(def i=0;i0){ + rets.add(dist) + } + println totalCount.get() +" "+ ret.size() + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + }; + }) + } + latch.await() + return rets.size() + } + + + def findThirdPrice(dist){ + def q = new BasicDBObject() + q.put("city", dist.city) + q.put("region", dist.region) + q.put("name", dist.name) + q.put("s_date", new BasicDBObject('$gte', 202201)) + def dbcursor = AVERAGE_STATISTICS.find(q) + def prices = [] + if(dbcursor){ + for(def d : dbcursor){ + def set = d.keySet(); + Iterator it = set.iterator() + while(it.hasNext()){ + def key = it.next() + if(key && key.contains("_price") && key!="fangjia_price" && key!="last_price"){ + def price = d[key] + if(price){ + prices << price + } + } + } + } + } + return prices; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocStat.groovy new file mode 100644 index 000000000..d1e0db444 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocStat.groovy @@ -0,0 +1,328 @@ +package com.fangjia.script.chongqing + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.rabbitmq.client.impl.AMQImpl +import org.apache.lucene.queryparser.classic.QueryParser +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + + +class MingShenPocStat { + + static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + + private static final DBCollection district_sell_stat_coll = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com") + final static DBCollection CITY_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", MongoUtils.MongoDB_Domain_mongo); + + static main(args) { + def task = new MingShenPocStat() +// task.run() + task.officeRun() + } + + def officeRun(){ + def path = "D:\\数据\\民生\\抽样-写字楼类.xlsx" + def datas = [] + ExcelUtils.modify(path, { d -> + if(d['B']){ + + def office = CommonDatautils.findDistrictById(d['C']) + if(office){ + def alias = [] + alias << office.name + if(office.alias){ + def nameAlias = office.alias + for(def o : nameAlias){ + if(o){ + alias << o + } + } + } + def aliasStr = escapeExprSpecialWord(alias.join("','")) + def houses = findOfficeHouseCount(office.city, office.region, aliasStr) + if(houses){ + if(houses.size()>=2){ + houses = houses[0..1] + } + for(def h : houses){ + def obj = findOfficeAvgPrice(office) + def rets = [] + rets << d['A'] + rets << d['B'] + rets << '' + rets << h.region + rets << office.address + rets << h.area + rets << h.floor + rets << h.height + def level = office.bed_kind + rets << level + rets << h.fitment + def date = office.finish_building_date + if(date){ + rets << date.substring(0, 4) + }else{ + rets << "" + } + + rets << "" + rets << "" + def price = "" + if(obj && obj.avg_price){ + price = obj.avg_price + rets << obj.avg_price + }else{ + rets << "" + } + + if(price){ + rets << (price*h.area).toInteger() + } + datas << rets + } + } + }else{ + def rets = [] + rets << d['A'] + rets << d['B'] + datas << rets + } + + // def districts = CommonDatautils.findAllDistrictByCity(d['C'],"office",["city", "region", "name", "alias","address","bed_kind","finish_building_date"]) + + } + }) + + ExcelUtils.write("D:\\数据\\民生\\抽样-写字楼类-样本.xlsx", datas, "样本") + } + + def run(){ + def path = "D:\\数据\\民生\\样本字段覆盖率.xlsx" +// def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Downloads\\二期估值数据每周统计\\二期估值数据每周统计\\估值数据统计结果20230112.xlsx", 0 , 1) +// +// ExcelUtils.modify(path, { row -> +// if(row['C']) { +// def total = 0 +// def pre = 0 +// for(def d : datas){ +// if(row['C']==d['A']){ +// total = d['B'] +// pre = d['H'] +// break +// } +// } +// if(total){ +// def c = (total*pre/100).toInteger() +// row['E'] = c +// row['F'] = pre+"%" +// } +// } +// }) + + def totalcount = findHouseTotal("大连") + def count = findHouseCount("大连", "height") + + println DataHandleUtils.getDoubleRound(2, count*100/totalcount) + +// ExcelUtils.multiThreadingModify(path, { row -> +// if(row['C']) { + //def districts = CommonDatautils.findAllDistrictByCity(row['C'],"district",["city", "region", "name", "alias"]) +// def districts = CommonDatautils.findDistrictByCityAndRegions("大连","district",["旅顺口"], ["city", "region", "name", "alias"]) +// def total = districts.size() +// if(total){ +// def house_dist_count = findHouseStat(districts) +// def pre = house_dist_count*100/total +// pre = DataHandleUtils.getDoubleRound(2, pre) +// println total+" "+house_dist_count+" "+pre +// } +// def count = findHouseCount(row['C']) +// row['D'] = count +// println row['C']+" "+count +// row['H'] = findDistrictSellStat(row['C']) +// row['G'] = getNewDist(row['C']) + +// def totalcount = findHouseTotal(row['C']) + +// def total = row['Z'] +// def count = findHouseCount(row['C'], "build_date") +// row['Q'] = DataHandleUtils.getDoubleRound(2, count*100/total) +// println count +" "+row['Q'] +// def obj = getFieldPrecent(row['C']) +// if(obj){ +// def detail = obj.district_detail +// row['E'] = detail.alias.toDouble()*100 +// row['F'] = detail.address.toDouble()*100 +// row['G'] = detail.lng.toDouble()*100 +// row['H'] = detail.buildType.toDouble()*100 +// row['I'] = detail.outPics.toDouble()*100 +// row['J'] = detail.developer.toDouble()*100 +// row['Q'] = detail.buildDate.toDouble()*100 +// row['R'] = detail.buildDate.toDouble()*100 +// } +// } +// }); + } + + def getFieldPrecent(city){ + def q = new BasicDBObject() + q.put("city", city) + def rets = CITY_INFO_CONN.find(q).sort(new BasicDBObject('weekend', -1)) + if(rets){ + return rets[0] + } + return null + } + + + def getNewDist(city){ + String sql = "select count(*) as count from community where city='${city}' and source='贝壳找房-新房' " + def count = null + ElasticUtils.executeSqlQuery(sql,{ data-> + count = data.count + }) + return count; + } + + def findDistrictSellStat(def city){ + def q = new BasicDBObject() + q.put("city", city) + def count = district_sell_stat_coll.count(q) + return count + } + + + def findHouseStat(districts){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + println "小区量:"+districts.size() + + def rets = new CopyOnWriteArrayList() + + for(def i=0;i0){ + rets.add(rMap) + } + println totalCount.get() +" "+ ret.size() + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + }; + }) + } + latch.await() + return rets.size() + } + + def findHouseTotal(city){ + def count = 0 + ElasticUtils.executeSqlQuery("select count(*) as count from seawater where p_date >= 20220101 and city='${city}'", { + count = it.count + println("$city\t${it.count}") + }) + return count + } + + def findHouseCount(city, field){ + def count = 0 + def sql = "select count(*) as count from seawater where city='${city}' and cat='sell' and "+field+" is not null" + ElasticUtils.executeSqlQuery(sql, { + count = it.count + println("$city\t${it.count}") + }) + return count + } + + def findHouses(def city, def region, def cat, def startDate, def name, def alias){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(false).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range) + + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + + + if(name.contains("弄小区")){ + def newName = name.replace("弄小区","弄") + if(!alias.contains(newName)){ + alias << newName + } + + } + + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + + def rets = [] + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + return houses; + } + + def findOfficeAvgPrice(office){ + def sql = "select * from office where city='${office.city}' and region='${office.region}' and name='${office.name}'" + def rets = [] + ElasticUtils.executeSqlQuery(sql, {d-> + rets << d + }) + if(rets){ + def obj = rets[0] + return obj + } + return null + + } + + def findOfficeHouseCount(city, region, names){ + def sql = "select * from seawater where city='${city}' and region='${region}' and cat='office_sell' and district_name in ('${names}') and root='58.com'" + def rets = [] + ElasticUtils.executeSqlQuery(sql, {d-> + rets << d + }) + return rets + } + + + /** + * 转义字符串中的特殊字符 + * 仅过滤Lucene regexps, 其他的勿用 + * @param keyword + * @return + */ + public static String escapeExprSpecialWord(String keyword) { + return QueryParser.escape(keyword); + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocTest.groovy new file mode 100644 index 000000000..112de372f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MingShenPocTest.groovy @@ -0,0 +1,459 @@ +package com.fangjia.script.chongqing + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.BaiduMapUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +class MingShenPocTest { + + def total = 5000 + static main(args) { + def task = new MingShenPocTest() + task.run() + } + def run(){ + def path = "D:\\数据\\民生\\poc测试.xlsx" + //9级标准化 + //structureV2(path)//针对海南poc地址数据此接口不合适使用 + //省市区拆分 +// ner(path)//城市,区域带市带区 +// match(path) +// addPrice(path) +// addLatLng(path) +// addAroundPrice(path) +// addAuthor(path) + addHouse(path) + } + + def structureV2(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(row['A']){ + def jsonObj = FangjiaClientUtils.addressStructure(null, row['A']) + if(jsonObj && jsonObj.result){ + + if(jsonObj.result.province){ + row['B'] = jsonObj.result.province + } + + if(jsonObj.result.city){ + def cityStr = null + if(jsonObj.result.city=="市辖区"){ + cityStr = jsonObj.result.province + }else{ + cityStr = jsonObj.result.city + } + row['C'] = RegionData.formatCity(cityStr) + } + + if(jsonObj.result.district){ + row['D'] = jsonObj.result.district + } + } + } + }) + } + + def ner(path){ + ExcelUtils.multiThreadingModify(path, { d -> + if(d['A'] && !d['B']) { + def json = FangjiaClientUtils.addressStand(null, d['A']) + if(json && json.entities){ + def entities = json.entities + for(def m : entities){ + if(m.type=="prov"){ + d['B'] = m.word + } + if(m.type=="city"){ + def cityStr = m.word + if(cityStr){ + d['C'] = cityStr//RegionData.formatCity(cityStr) + } + } + if(m.type=="dist") { + if(d['C']){ + d['D'] = m.word //RegionData.formatRegion(d['C'], m.word) + } + } + } + } + } + }); + } + + //可以根据测试数据添加 推荐楼盘名 + def match(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(row['C'] && !row['E']){ + def city = RegionData.formatCity(row['C']) + def region = null + if(city){ + region = RegionData.formatRegion(city, row['D']) + } + def address = row['A'] + println "[match] city:"+city+" region:"+region+" address:"+address + def jsonObject = OpenApiClient.addressMatch(city, address, null, false, region, null)//"property" + if(jsonObject != null && jsonObject.precise!=-1) { + row['E'] = jsonObject.get("city") + row['F'] = jsonObject.get("district") + row['G'] = jsonObject.get("name") + row['H'] = jsonObject.get("propertyType") + row['I'] = jsonObject.get("id") + } + } + }) + } + + def addPrice(path){ + ExcelUtils.multiThreadingModify(path, { row -> + def commid = row['I'] + if(commid && !row['J']){ + def dist = CommonDatautils.findDistrictById(commid.trim()) + if(dist){ + def house = [:] + house.area = 100 + def price = getEvalutaionPrice(house, dist) + if(price){ + row['J'] = price + } +// def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name) +// if(priceObj){ +// row['J'] = priceObj.avg_price +// } + } + } + }) + } + + def getEvalutaionPrice(house, dist){ + def params = [:] + params.put("city", dist.city); + params.put("district", dist.region); + params.put("name", dist.name); + params.put("size", String.valueOf(house.area)); + if(house.height) { + params.put("maxFloor", dist.house_height.toString()); + params.put("direction", getDirectionCode(house.direction).toString());//朝向 + if (house.house_floor) { + params.put("floor", house.house_floor.toString()); + } + } + def fitmentCode = null + if(house.fitment){ + fitmentCode = getFitmentCode(house.fitment) + params.put("decoration", fitmentCode.toString());//装修 + } + params.put("scope", "1") + def result = OpenApiClient.evaluateDistrictPrice(params); + + if(result){ + return result.avgPrice + } + return null + } + + + def getDirectionCode(def s){ + def directMap = ["西南":1, "东南":2, "南":3, "西北":4, "东北":5, "东":6, "西":7, "北":8, "东西":9, "南北":10] + return directMap.get(s); + } + def getFitmentCode(def s){ + if(s){ + if(["毛坯"].contains(s)){ + return 1; + }else if(["简单装修","简装"].contains(s)){ + return 2; + }else if(["精装修","精装"].contains(s)){ + return 3; + }else if(["豪华装修","豪装"].contains(s)){ + return 4; + } + } + return null + } + + def addLatLng(path){ + ExcelUtils.multiThreadingModify(path, { d -> + if(!d['J'] && d['H'] && (d['H']=="普通住宅" || d['H']=="商住" || d['H']=="别墅")) { + def json = BaiduMapUtils.searchByAddress(d['A']) + if(json && json.location){ + d['K'] = json.location.lng + d['L'] = json.location.lat + } + } + }) + } + + + def addAroundPrice(path){ + ExcelUtils.multiThreadingModify(path, { d -> + if(!d['J'] && d['C'] && d['K']){ + def city = RegionData.formatCity(d['C']) + def dist = new BasicDBObject() + dist.city = city + dist.lng = d['K'].toString().toDouble()*10000000000L + dist.lat = d['L'].toString().toDouble()*10000000000L + dist.estateType = "普通住宅"//d['H'] + if(dist.lat) { + List list = DbUtils.findAroundDistrictByRadius(dist, 1000) + if (list) { + list = sortByDistance(dist, list); + if (list && list.size() > 10) { + list = list[0..9] + } + if (list) { + def prices = [] + for (def s : list) { + def priceObj = CommonDatautils.findDistrictPrice(s.city, s.region, s.name) + if (priceObj) { + prices << priceObj.avg_price + } + } + if (prices) { + def arround_price = (int) (prices.sum() / prices.size()) + d['J'] = arround_price + } + } + } + } + } + }); + } + + def sortByDistance(d, list){ + for(def nb : list){ + def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0) + nb.distance = distance + } + //排序 + Collections.sort(list, new Comparator() { + @Override + int compare(Object o1, Object o2) { + if(o1.distance > o2.distance) + return 1;//第二个元素(o1)比第一个元素(o2)大,返回-1 + if(o1.distance == o2.distance) + return 0; + return -1; + } + }) + return list; + } + + + def addAuthor(def path){ + def authors = ["董彦均","曹玉彦","付贵生","沈燕芳","Catherine", "王蕾","吉珉", "张宝剑"] + def personTotal = authors.size() + def avg = Math.ceil(total/personTotal) + + def index = 0 + def count = 0; + ExcelUtils.modify(path, { row -> + if(row['A']){ + if(index < avg){ + row['M'] = authors[count] + } + index++ + if(index >= avg){ + count++ + index = 0 + } + } + }); + } + + def addHouse(path){ + def datas = [] + ExcelUtils.multiThreadingModify(path, { row -> + def commId= row['E'] + if(commId){ + def dist = CommonDatautils.findDistrictById(commId) + if(dist){ + def rets = findHouses(row['H'], row['I'], "sell", 20200101,20230113, row['J'], null, row['k']) + if(rets){ + def data = [] + for(def r : rets){ + data << r.city + data << r.region + data << r.district_name + data << r.floor + data << r.height + data << r.fitment + data << r.direction + data << r.build_type + data << r.area + data << r.avg_price + datas << data + } + } + } + } + }); + + ExcelUtils.write("D:\\数据\\民生\\民生样本数据.xlsx", datas, "样本数据") + } + + def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range) + + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + + + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + + def rets = [] + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + + if(houses){ + for (int k=0;k60){ + continue; + } + +// if(!seawater.room){ +// continue; +// } +// if(!seawater.room){ +// seawater.room = null +// } + + if(seawater.room){ + if(seawater.room>=11 || seawater.room<1){ + continue; + } + } + + if(!seawater.hall){ + seawater.hall = null + } + if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){ + continue; + } + if(!seawater.toilet){ + seawater.toilet = null + } + if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){ + continue; + } + + + def floor = parseFloor(seawater.floor_info) + if(seawater.height && floor && floor.isNumber()){ + if(floor.toInteger()>seawater.height){ + floor = null + seawater.height = null + } + + } + + if(!floor){ + seawater.floor = null + } + if(floor && floor.toInteger()<1){ + floor = null + } + + def rowMap = [:]; + + rowMap.houseId = seawater.id +// rowMap.city = CcbCommonUtils.getCcbCity(city) + //rowMap.region =seawater.region + //rowMap.address = seawater.address + rowMap.area = seawater.area + rowMap.address = seawater.address + rowMap.total_price = seawater.total_price + rowMap.avg_price = seawater.avg_price + rowMap.build_type = seawater.build_type + rowMap.build_date = seawater.build_date + rowMap.district_name = seawater.district_name + rowMap.height = seawater.height=="null"?null:seawater.height + if(rowMap.height && rowMap.height<1){ + rowMap.height = null + } + rowMap.floor = floor=="null"?null:floor + rowMap.room = seawater.room=="null"?null:seawater.room + rowMap.hall = seawater.hall=="null"?null:seawater.hall + rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet + + + def direction = parseDirection(seawater.direction) + rowMap.direction = direction=="null"?null:direction + + def fitment = parseFitment(seawater.fitment) + rowMap.fitment = fitment=="null"?null:fitment + + + + if(seawater.p_date instanceof String){ + seawater.p_date = seawater.p_date.toInteger() + } + + rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") + rowMap.c_date = DateUtils.getConverDate(seawater.m_date) + rowMap.m_date = DateUtils.getConverDate(seawater.m_date) + + rets << rowMap + } + } + return rets; + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/ThreeDESUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/ThreeDESUtils.groovy new file mode 100644 index 000000000..3d685a1a5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/ThreeDESUtils.groovy @@ -0,0 +1,105 @@ +package com.fangjia.script.chongqing + +import com.fangjia.script.util.ExcelUtils +import org.apache.commons.io.FileUtils + +import javax.crypto.Cipher; +import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; +import java.util.Base64; + +/** + * 3DES加密工具类 + * @author qzz + */ +public class ThreeDESUtils { + + /** + * 加解密统一编码方式 + */ + private final static String ENCODING = "utf-8"; + + /** + * 加解密方式 + */ + private final static String ALGORITHM = "DESede"; + + /** + *加密模式及填充方式 + */ + private final static String PATTERN = "DESede/ECB/pkcs5padding"; + + /** + * 3DES加密 + * + * @param plainText 普通文本 + * @param sK 秘钥(24位密码) + * @return + * @throws Exception + */ + public static String encode(String plainText,String sK) throws Exception { + SecretKey secretKey = new SecretKeySpec(sK.getBytes(ENCODING), ALGORITHM); + // 3DES加密采用pkcs5padding填充 + Cipher cipher = Cipher.getInstance(PATTERN); + // 用密匙初始化Cipher对象 + cipher.init(Cipher.ENCRYPT_MODE, secretKey); + // 执行加密操作 + byte[] encryptData = cipher.doFinal(plainText.getBytes(ENCODING)); + return Base64.getEncoder().encodeToString(encryptData); + } + + /** + * 3DES解密 + * + * @param encryptText 加密文本 + * @return + * @throws Exception + */ + public static String decode(String encryptText, String sK) throws Exception { + SecretKey secretKey = new SecretKeySpec(sK.getBytes(ENCODING), ALGORITHM); + // 3DES加密采用pkcs5padding填充 + Cipher cipher = Cipher.getInstance(PATTERN); + + // 用密匙初始化Cipher对象 + cipher.init(Cipher.DECRYPT_MODE, secretKey); + // 正式执行解密操作 + byte[] decryptData = cipher.doFinal(Base64.getDecoder().decode(encryptText)); + return new String(decryptData, ENCODING); + } + + public static void main(String[] args) throws Exception { + + def rets = [] + def datas = readTxt("D:\\数据\\民生\\抽样-住宅类(普通住宅、公寓).txt") + def index = 0 + for(def d : datas){ + index++ + def row = [] + def str = decode(d,"111036369260679051122113") + println str + def id = str.substring(str.length()-4) + row << id + row << str.substring(0, str.indexOf(id)) + rets << row + } + ExcelUtils.write("D:\\数据\\民生\\民生银行测试数据.xlsx", rets, "抽样-住宅类(普通住宅、公寓)") + +// ExcelUtils.multiThreadingModify("D:\\数据\\民生\\加密数据.xlsx", { row -> +// //加密 +// String str = row['A'] +// //解密 +// row['B'] = (decode(str,"111036369260679051122113")); +// }); + + } + + public static def readTxt(def file){ + def lines = FileUtils.readLines(new File(file)) + def ret = []; + for(line in lines) { + ret << line; + } + return ret; + } +} + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy index 088626285..1b604d376 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy @@ -18,6 +18,7 @@ import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Pattern //小区月度均价表(带涨跌幅控制) @BatchOwner(Watcher.GaoJuan) @@ -78,6 +79,7 @@ class DistrictSellStatBatch extends BaseBatch { }else{ districts = DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000) } + print city +" "+districts.size() def weekendPriceMap = findPriceMap(city, lastMonthWeekend)//上个月小区均价 def curMonthlyPriceMap = findMonthlyPrice(city, curMonth)//当月小区月均价 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index 7e4a0bd2c..44e4dd54c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202211.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202301.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy index 577aac71d..7a531b0e7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy @@ -51,6 +51,7 @@ class AddAvgpriceBatch { for(def disKey : res2){ def array = disKey.split("##"); def estateType2 = districtKeysMap.get(disKey) + //发重算均价命令 CommonUtils.recalculationAverage(array[0], array[1], array[2], estateType2, 0, new Date()) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ShLongShangHangLocation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ShLongShangHangLocation.groovy new file mode 100644 index 000000000..4cc9588c1 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ShLongShangHangLocation.groovy @@ -0,0 +1,52 @@ +package com.fangjia.script.gj + +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils + +class ShLongShangHangLocation { + static main(args) { + def task = new ShLongShangHangLocation() + task.run() + } + + def run(){ + def rets = ["上海市黄兴路1599号","上海市杨浦区长阳路1687号A栋","上海市杨浦区佳木思路432号","上海市杨浦区控江路1063号","上海市杨浦区鞍山路100号","上海市杨浦区延吉东路287号","上海市杨浦区殷行路408号","上海市杨浦区闸殷路1555号102室","上海市杨浦区平凉路1500号"] + def map = [:] + for(def r : rets){ + def location = getLocation(r) + if(location){ + map.put(r, location) + } + } + def titleMap = [0:'C',1:'D',2:'E',3:'F',4:'G',5:'H',6:'I',7:'J',8:'K'] + ExcelUtils.modify("D:\\数据\\上海农商行\\网点与居委分布20230118.xlsx",{row-> + if(row['B'] && !row['B'].contains("地址") && row['B'].contains('齐齐哈尔路')){ + for(def i=0;i OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20221101", "yyyyMMdd") -def endDate = DateUtil.parseDate("20221201", "yyyyMMdd") +def startDate = DateUtil.parseDate("20230101", "yyyyMMdd") +def endDate = DateUtil.parseDate("20230201", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 @@ -38,6 +38,6 @@ while (startDate.before(endDate)) { startDate = nextDate } -ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行.xlsx", data, "202211") +ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, "202301") From 72d565c7606fa940da4f11508b85c7c4091316c4 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 17 Feb 2023 15:12:07 +0800 Subject: [PATCH 20/30] update --- .../BigdataCommunityBatch.groovy | 3 +- .../ccbevaluation/BigdataForsaleBatch.groovy | 48 +++++++++++++++++-- .../ccbevaluation/StagingDistrictBatch.groovy | 37 ++++++++++++++ 3 files changed, 81 insertions(+), 7 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 123ada0cf..0177411a5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -41,7 +41,6 @@ class BigdataCommunityBatch extends BaseBatch{ @Override public void process(Object args) { def citys = CcbCommonUtils.getRunningCitys(args) - citys = ["上海"] def rows = [] def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途", @@ -251,7 +250,7 @@ class BigdataCommunityBatch extends BaseBatch{ sqlInstance.withBatch(20000, sql){ ps-> for(def r : rows) { ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], - r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], null, '2023-02-28', null, '2999-12-31', null, null, null) + r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38]) ps.toString() } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 7eac9b9f7..f90b1cfdc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -57,9 +57,13 @@ class BigdataForsaleBatch extends BaseBatch{ public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"]; public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60) + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/postgres?stringtype=unspecified", + "postgres", "postgres", "org.postgresql.Driver") + static main(args) { runBatch(new BigdataForsaleBatch(), args); } @@ -69,6 +73,7 @@ class BigdataForsaleBatch extends BaseBatch{ //目前查询库中是否已存在,用的是Oracle def index = 0 def citys = CcbCommonUtils.getRunningCitys(args) + citys = ["福州"] def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ index++ @@ -181,7 +186,9 @@ class BigdataForsaleBatch extends BaseBatch{ // headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"] // } FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true); - println "txt文件生成 " + println "txt文件生成->手动导入到Oracle " + + saveToPgSql(allDatas) } @@ -610,9 +617,9 @@ class BigdataForsaleBatch extends BaseBatch{ def count = isExistFromHouse(seawater.id) if(count){ - rets << rowMap - }else{ println seawater.id +" 已存在" + }else{ + rets << rowMap } } @@ -812,9 +819,9 @@ class BigdataForsaleBatch extends BaseBatch{ def count = isExistFromHouse(seawater.id) if(count){ - rets << rowMap - }else{ println seawater.id +" 已存在" + }else{ + rets << rowMap } } } @@ -1019,4 +1026,35 @@ class BigdataForsaleBatch extends BaseBatch{ return count; } + def saveToPgSql(rows){ + for(def r : rows) { + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ + r[i] = null + } + } + r << null + r << '2023-02-28' + r << null + r << '2999-12-31' + r << null + r << null + r << null + } + + println "start: "+new Date() + def sql = "insert into base.t0831_bigdata_forsale_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + pgsqlInstance.withBatch(20000, sql){ ps-> + for(def r : rows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], + r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39], r[40], + r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49], r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58]) + ps.toString() + } + } + + println "end: "+new Date() + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy index ed95e4cc6..d11ca41ad 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy @@ -9,6 +9,7 @@ import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject +import groovy.sql.Sql import org.bson.types.ObjectId /** @@ -18,6 +19,9 @@ import org.bson.types.ObjectId @BatchOwner(Watcher.GaoJuan) class StagingDistrictBatch extends BaseBatch{ + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified", + "postgres", "postgres", "org.postgresql.Driver") + static main(args) { runBatch(new StagingDistrictBatch(), args); } @@ -47,6 +51,8 @@ class StagingDistrictBatch extends BaseBatch{ //ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\COMMUNITY_STAGING_GJ.xlsx", results, "数据") FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); + saveToPgSql(rows) + } def createCityData(def dists, def fenQiMap){ @@ -86,4 +92,35 @@ class StagingDistrictBatch extends BaseBatch{ } return fenQiMap; } + + + def saveToPgSql(rows){ + for(def r : rows) { + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ + r[i] = null + } + } + r << null + r << '2023-02-28' + r << null + r << '2999-12-31' + r << null + r << null + r << null + } + + println "start: "+new Date() + def sql = "insert into base.t0831_community_staging_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + pgsqlInstance.withBatch(20000, sql){ ps-> + for(def r : rows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], + r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38]) + ps.toString() + } + } + + println "end: "+new Date() + } } From 5e3d4fe6cad9f668978498d5680c12563ed43096 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 17 Feb 2023 15:20:32 +0800 Subject: [PATCH 21/30] =?UTF-8?q?=E5=AD=97=E6=AE=B5=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/ccbevaluation/StagingDistrictBatch.groovy | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy index d11ca41ad..899243d11 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy @@ -111,12 +111,11 @@ class StagingDistrictBatch extends BaseBatch{ } println "start: "+new Date() - def sql = "insert into base.t0831_community_staging_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + def sql = "insert into base.t0831_community_staging_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?)" pgsqlInstance.withBatch(20000, sql){ ps-> for(def r : rows) { - ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], - r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38]) + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13]) ps.toString() } } From 470f2257d048ab4b925e19fb3639cb2c118c4e19 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Fri, 17 Feb 2023 15:29:26 +0800 Subject: [PATCH 22/30] =?UTF-8?q?=E8=A7=84=E8=8C=83=E7=89=A9=E4=B8=9A?= =?UTF-8?q?=E7=B1=BB=E5=9E=8B=E8=BE=93=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/open/controllers/AddressController.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index 27942a8b1..f998411a0 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -523,6 +523,13 @@ public class AddressController { } MatchDistrictResult matchResult = addressMatchProxyService.checkMatchDistrictCorrect(param); jsonResult.put("matchResult", matchResult); + if (jsonResult.containsKey("category")) { + if ("office".equals(jsonResult.getString("category"))) { + jsonResult.put("propertyType", "写字楼"); + } else if ("shop".equals(jsonResult.getString("category"))) { + jsonResult.put("propertyType", "商铺"); + } + } } else if (jsonResult != null) { // 判断是否为自然村 boolean matchedVillage = false; From 1f0bacd013cf4421e19a3153619f6c810f3f9f34 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 17 Feb 2023 17:11:19 +0800 Subject: [PATCH 23/30] update --- .../ccbevaluation/BigdataDealBatch.groovy | 37 ++++++++ .../ccbevaluation/BigdataForsaleBatch.groovy | 85 +++++++++++++------ 2 files changed, 94 insertions(+), 28 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index fe8a32a3c..d2cbed0d1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -50,6 +50,9 @@ class BigdataDealBatch extends BaseBatch{ def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified", + "postgres", "postgres", "org.postgresql.Driver") + static main(args) { runBatch(new BigdataDealBatch(), args); } @@ -106,6 +109,8 @@ class BigdataDealBatch extends BaseBatch{ // // FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); // println "txt 生成 " + rows.size() + + saveToPgSql(rows) } def createData(def rets, cityCodeDatas, distCodeMap, source){ @@ -680,4 +685,36 @@ class BigdataDealBatch extends BaseBatch{ latch.await() println city +" "+trades.size()+" "+new Date().toString() } + + def saveToPgSql(rows){ + for(def r : rows) { + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ + r[i] = null + } + } + r << null + r << '2023-02-28' + r << null + r << '2999-12-31' + r << null + r << null + r << null + } + + println "start: "+new Date() + def sql = "insert into base.t0831_bigdata_deal_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + sqlInstance.withBatch(20000, sql){ ps-> + for(def r : rows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], + r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39], + r[40], r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49],r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58], r[59], + r[60], r[61], r[62], r[63], r[64], r[65], r[66], r[67], r[68], r[69],r[70], r[71]) + ps.toString() + } + } + + println "end: "+new Date() + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index f90b1cfdc..0f85fe67b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -122,7 +122,8 @@ class BigdataForsaleBatch extends BaseBatch{ totalCount.addAndGet(1) def commId = rMap.get("_id").toString() - def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias, rMap.estate_type2) + + def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias, rMap.estate_type2, commId) //def ret = findHousesByDistId(city, commId, "sell", startDate,endDate) // if(ret && ret.size()<100){//导历史房源需要,以后不需要 // ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias) @@ -435,12 +436,24 @@ class BigdataForsaleBatch extends BaseBatch{ } - def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){ + def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, def commId){ def rets1 = findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2) def rets2 = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1) rets1.addAll(rets2) - return rets1; + if(rets1){ + def newRets = [] + def houseIds = getAllHouseIds(commId) + for(def r : rets1){ + if(!houseIds.contains(r.houseId)){ + newRets << r + }else{ + println r.houseId +" 已存在" + } + } + return newRets + } + return []; } def findHousesByPdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){ @@ -474,8 +487,11 @@ class BigdataForsaleBatch extends BaseBatch{ } + def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"] + def index = new Random().nextInt(6); + def rets = [] - def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder); if(houses){ for (int k=0;k - distIds << it.commid + houseIds << it.FORSALEID }); - return distIds; + return houseIds; } + def isExistFromHouse(def houseId){ def sql = "SELECT count(FORSALEID) as count FROM bigdata_forsale_gj WHERE FORSALEID = '"+houseId+"'"; From 17d0dfafe5289deeafd2fb7a76ffe01f87c21edd Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 17 Feb 2023 17:38:35 +0800 Subject: [PATCH 24/30] =?UTF-8?q?=E6=9F=A5=E6=88=BF=E6=BA=90=E5=8A=A0?= =?UTF-8?q?=E5=9F=8E=E5=B8=82id?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/BigdataForsaleBatch.groovy | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 0f85fe67b..cf248fb61 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -123,7 +123,7 @@ class BigdataForsaleBatch extends BaseBatch{ def commId = rMap.get("_id").toString() - def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias, rMap.estate_type2, commId) + def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias, rMap.estate_type2, commId, cityCodeDataMap) //def ret = findHousesByDistId(city, commId, "sell", startDate,endDate) // if(ret && ret.size()<100){//导历史房源需要,以后不需要 // ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias) @@ -436,14 +436,15 @@ class BigdataForsaleBatch extends BaseBatch{ } - def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, def commId){ + def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, def commId, def cityCodeDataMap){ def rets1 = findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2) def rets2 = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1) rets1.addAll(rets2) if(rets1){ def newRets = [] - def houseIds = getAllHouseIds(commId) + def cityCode = cityCodeDataMap.get(city) + def houseIds = getAllHouseIds(commId, cityCode) for(def r : rets1){ if(!houseIds.contains(r.houseId)){ newRets << r @@ -1034,8 +1035,8 @@ class BigdataForsaleBatch extends BaseBatch{ println ids.size() } - def getAllHouseIds(def commid){ - def sql = "SELECT FORSALEID FROM bigdata_forsale_gj WHERE commid = '"+commid+"'"; + def getAllHouseIds(def commid, def cityCode){ + def sql = "SELECT FORSALEID FROM bigdata_forsale_gj WHERE cityid='"+cityCode+"' and commid = '"+commid+"'"; def houseIds = new HashSet() sqlInstance.eachRow(sql,{it-> From 05673e6b2c34dd4176ab61be30519e23d30ee28a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 17 Feb 2023 18:18:15 +0800 Subject: [PATCH 25/30] update --- .../com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index cf248fb61..007fc2b86 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -73,7 +73,6 @@ class BigdataForsaleBatch extends BaseBatch{ //目前查询库中是否已存在,用的是Oracle def index = 0 def citys = CcbCommonUtils.getRunningCitys(args) - citys = ["福州"] def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ index++ From c83a6d13696ad2f8e2b88753369663789ce2bcf5 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 19 Feb 2023 10:54:55 +0800 Subject: [PATCH 26/30] =?UTF-8?q?=E6=88=BF=E6=BA=90=E5=8E=BB=E9=87=8D?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/BigdataForsaleBatch.groovy | 102 +++++++++++++----- .../gj/paimai/BeiKeCrawlByChrome.groovy | 4 +- .../gj/paimai/LianjiaCrawlByChrome.groovy | 2 +- 3 files changed, 76 insertions(+), 32 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 007fc2b86..74740ef87 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -21,6 +21,7 @@ import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors @@ -107,6 +108,33 @@ class BigdataForsaleBatch extends BaseBatch{ def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate){ + def retMap = findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate) + println city + "(原)房源总量:"+retMap.size() + + def allDatas = retMap.values() + allDatas = repeatHouseId(allDatas) + println city + "(新)房源总量:"+allDatas.size() + + + def fileName = "/home/goojia/ccb/BIGDATA_FORSALE_GJ.dat" + if(CommonDatautils.isWindow()){ + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_FORSALE_GJ.dat" + } + + + // def allDatas = createSellDatas(city, retMap.values()); + + def headers = [] +// if(index == 1){ +// headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"] +// } + FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true); + println "txt文件生成->手动导入到Oracle " + + saveToPgSql(allDatas) + } + + def findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate){ def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(districts.size()) println city +" 小区量:"+districts.size() @@ -172,23 +200,38 @@ class BigdataForsaleBatch extends BaseBatch{ }) } latch.await() + return retMap + } - def fileName = "/home/goojia/ccb/BIGDATA_FORSALE_GJ.dat" - if(CommonDatautils.isWindow()){ - fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_FORSALE_GJ.dat" + + def repeatHouseId(allDatas){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(allDatas.size()) + def rets = new CopyOnWriteArrayList() + for(def i=0;i手动导入到Oracle " - - saveToPgSql(allDatas) + latch.await() + return rets } @@ -440,20 +483,21 @@ class BigdataForsaleBatch extends BaseBatch{ def rets1 = findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2) def rets2 = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1) rets1.addAll(rets2) - if(rets1){ - def newRets = [] - def cityCode = cityCodeDataMap.get(city) - def houseIds = getAllHouseIds(commId, cityCode) - for(def r : rets1){ - if(!houseIds.contains(r.houseId)){ - newRets << r - }else{ - println r.houseId +" 已存在" - } - } - return newRets - } - return []; +// if(rets1){ +// def newRets = [] +// def cityCode = cityCodeDataMap.get(city) +// def houseIds = getAllHouseIds(commId, cityCode) +// for(def r : rets1){ +// if(!houseIds.contains(r.houseId)){ +// newRets << r +// }else{ +// println r.houseId +" 已存在" +// } +// } +// return newRets +// } +// return []; + return rets1; } def findHousesByPdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 0bab915c0..f641c214c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -401,13 +401,13 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = e.attr("href") - if(e.text().contains("福州")){ +// if(e.text().contains("福州")){ def cityMap = [:] cityMap.city = e.text() cityMap.url = "https:"+href list << cityMap // cityMap.put(e.text(), "https:"+href) - } +// } } } return list diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 94a13d8e8..e82b8bc45 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -478,7 +478,7 @@ public class LianjiaCrawlByChrome { if(cityEles){ for(def e : cityEles) { def href = e.attr("href") - if (!href.contains("fang.lianjia") && citys.contains(e.text())){ + if (!href.contains("fang.lianjia")){ cityMap.put(e.text(), href) } } From 2cbd678451f8e6ee38627f768f1806dffb225306 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 19 Feb 2023 11:41:28 +0800 Subject: [PATCH 27/30] upate --- .../com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 74740ef87..5b075b74f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -217,7 +217,7 @@ class BigdataForsaleBatch extends BaseBatch{ totalCount.addAndGet(1) def count = isExistFromHouse(houseInfo[1]) if(count){ - println houseInfo[1]+" 已存在" + println houseInfo[1]+" 已存在 "+totalCount.get() }else{ rets << houseInfo } From 7d7c9daac21084fe18d17d582be0f25167e7da06 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 20 Feb 2023 10:52:33 +0800 Subject: [PATCH 28/30] =?UTF-8?q?pgsql=20=E6=8D=A2=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/BigdataDealBatch.groovy | 3 +- .../ccbevaluation/BigdataForsaleBatch.groovy | 6 ++- .../ccbevaluation/GuiDingPriceBatch.groovy | 39 ++++++++++++++++-- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 40 +++++++++++++++++-- 4 files changed, 77 insertions(+), 11 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index d2cbed0d1..3c9e030bb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -705,13 +705,12 @@ class BigdataDealBatch extends BaseBatch{ println "start: "+new Date() def sql = "insert into base.t0831_bigdata_deal_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" - sqlInstance.withBatch(20000, sql){ ps-> + pgsqlInstance.withBatch(20000, sql){ ps-> for(def r : rows) { ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39], r[40], r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49],r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58], r[59], r[60], r[61], r[62], r[63], r[64], r[65], r[66], r[67], r[68], r[69],r[70], r[71]) - ps.toString() } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 5b075b74f..398fd08bd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -58,11 +58,13 @@ class BigdataForsaleBatch extends BaseBatch{ public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"]; public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60) + public static ExecutorService fixedThreadPool2 = Executors.newFixedThreadPool(100) + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") - def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/postgres?stringtype=unspecified", + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified", "postgres", "postgres", "org.postgresql.Driver") static main(args) { @@ -211,7 +213,7 @@ class BigdataForsaleBatch extends BaseBatch{ for(def i=0;i + for(def r : rows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20]) + } + } + + println "end: "+new Date() + } + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 8a2b1c55e..3aedc3aa5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202212//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202210 - def endDate = 202212 + def sDate= 202301//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202211 + def endDate = 202301 // def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") @@ -36,6 +36,10 @@ class ThirdPartyPriceBatch extends BaseBatch { def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified", + "postgres", "postgres", "org.postgresql.Driver") + static main(args) { runBatch(new ThirdPartyPriceBatch(), args); } @@ -69,6 +73,8 @@ class ThirdPartyPriceBatch extends BaseBatch { def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true); + saveToPgSql(rows) + } @@ -618,4 +624,32 @@ class ThirdPartyPriceBatch extends BaseBatch { } + def saveToPgSql(rows){ + for(def r : rows) { + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ + r[i] = null + } + } + r << null + r << '2023-02-28' + r << null + r << '2999-12-31' + r << null + r << null + r << null + } + + println "start: "+new Date() + def sql = "insert into base.t0831_bigdata_deal_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + pgsqlInstance.withBatch(20000, sql){ ps-> + for(def r : rows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18]) + } + } + + println "end: "+new Date() + } + } From 53d9032bba96b31074bd6c31f1116704a4c4ccdc Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 20 Feb 2023 10:57:36 +0800 Subject: [PATCH 29/30] 94->236 --- .../com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 908f4adf5..1d9ae5347 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -61,7 +61,7 @@ class CcbCommonUtils { "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") }else if(countStr=="252"){ citys = CcbCommonUtils.citys_252 - sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", + sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") } }else{ From 5d8b5abfc4b4f191122ef47c2b1bf8ea21cb6e2a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 20 Feb 2023 10:58:15 +0800 Subject: [PATCH 30/30] =?UTF-8?q?100-=E3=80=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 398fd08bd..82baa96ee 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -58,7 +58,7 @@ class BigdataForsaleBatch extends BaseBatch{ public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"]; public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60) - public static ExecutorService fixedThreadPool2 = Executors.newFixedThreadPool(100) + public static ExecutorService fixedThreadPool2 = Executors.newFixedThreadPool(200) def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",