From ba9322e206ba373e023c39e54c29913a358acb81 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 10 Feb 2023 16:41:04 +0800 Subject: [PATCH] =?UTF-8?q?=E8=BF=98=E5=8E=9F=E8=9E=8D=E5=90=88=E5=AF=BC?= =?UTF-8?q?=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ronghe/ExportDistrictRongHe.groovy | 67 +++++++++---------- .../gj/paimai/BeiKeCrawlByChrome.groovy | 41 ++++++------ .../gj/paimai/LianjiaCrawlByChrome.groovy | 38 ++++++----- 3 files changed, 76 insertions(+), 70 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 7c919169c..8215007e5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -38,35 +38,34 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //按天导(每天一个文件夹) -// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) + exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) + if(curDate.toString().endsWith("10")){ + exportBigdataCommunityGJRH()// + exportBigdataBuildingFJW()// + exportBigdataCommfeeGj()// - if(curDate.toString().endsWith("11")){ -// exportBigdataCommunityGJRH()// -// exportBigdataBuildingFJW()// -// exportBigdataCommfeeGj()// -// -// exportBigdataCommbuildstruGJ()// -// exportBigdataCommhousetypeGJ()// -// exportBigdataCommbuildyearGj()// -// exportBigdataCommhouseuseGJ()// -// exportBigdataPositionGJ()// -// exportBigdataDistrictFJW()// + exportBigdataCommbuildstruGJ()// + exportBigdataCommhousetypeGJ()// + exportBigdataCommbuildyearGj()// + exportBigdataCommhouseuseGJ()// + exportBigdataPositionGJ()// + exportBigdataDistrictFJW()// exportBigdataAliasFJW()// -// exportBigdataBuildprojFjw() -// exportBigdataCommlayoutpicFjw() -// exportEvaluateRate() -// -// exportBigdataCityPrice(queryDate) -// exportBigdataDistPrice(queryDate) -// exportBigdataCommPrice(queryDate) -// -// //因为融合暂不切库,所以需要加传一张第三方均价表到融合中, 从1月10日开始传输,长期供 -// def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); -// exportThirdPartyPrice2(thirdPartyMonth, new_dists) -// -// + exportBigdataBuildprojFjw() + exportBigdataCommlayoutpicFjw() + exportEvaluateRate() + + exportBigdataCityPrice(queryDate) + exportBigdataDistPrice(queryDate) + exportBigdataCommPrice(queryDate) + + //因为融合暂不切库,所以需要加传一张第三方均价表到融合中, 从1月10日开始传输,长期供 + def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); + exportThirdPartyPrice2(thirdPartyMonth, new_dists) + + //1月10日 新增5张表,导92城,增量数据,2月10日导344城,其中252城所有历史数据,92城增量,3月后可能不用提供数据 exportCommunityBasePrice(lastMonth) exportCommunityPdescribeGj(lastMonth) @@ -74,15 +73,15 @@ class ExportDistrictRongHe extends BaseBatch{ exportBigdataSupplierStatistics(lastMonth) exportBigdataModelingBase(lastMonth) + //2月8号 临时导出苏州一年的数据 +// exportBigdataForsale("2022-07-01", new_dists) //现在按updatetime导出 + } - //2月8号 临时导出苏州一年的数据 -// exportBigdataForsale("2022-07-01", new_dists) //现在按updatetime导出 + SSHHelper jschCommand = new SSHHelper(); + System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); -// SSHHelper jschCommand = new SSHHelper(); -// System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); -// -// exportOk(curDate) + exportOk(curDate) } @@ -1164,7 +1163,7 @@ class ExportDistrictRongHe extends BaseBatch{ for(def city : citys) { def cityCode = cityCodeMap.get(city) // def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";// - def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE where CITYID='${cityCode}' and YMID=${queryDate}";// + def sql = "select CITYID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_CITYPRICE_FJW where CITYID='${cityCode}' and YMID=${queryDate}";// sqlInstance.eachRow(sql,{r-> def data = [] @@ -1211,7 +1210,7 @@ class ExportDistrictRongHe extends BaseBatch{ def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select CITYID,DISTID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_DISTPRICE where CITYID='${cityCode}' and YMID=${queryDate}";// and YMID=${queryDate} + def sql = "select CITYID,DISTID, YMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_DISTPRICE_FJW where CITYID='${cityCode}' and YMID=${queryDate}";// and YMID=${queryDate} sqlInstance.eachRow(sql,{r-> def data = [] data << r.CITYID @@ -1258,7 +1257,7 @@ class ExportDistrictRongHe extends BaseBatch{ def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE where CITYID='${cityCode}' and YMID=${queryDate} ";// + def sql = "select CITYID, YMID,COMMID, SALEPRICE,RENTPRICE,SALECOUNT,RENTCOUNT, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, HOUSEUSE,SALEBLDGAREA,RENTBLDGAREA,SALETOTALPRICE,RENTTOTALPRICE,DATATYPE,SDATERANGE,RDATERANGE,SOURCECODE from BIGDATA_COMMPRICE_FJW where CITYID='${cityCode}' and YMID=${queryDate} ";// sqlInstance.eachRow(sql,{r-> def data = [] data << r.CITYID diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 24f03c7cc..f641c214c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -251,26 +251,29 @@ public class BeiKeCrawlByChrome { detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd") } detail.source = "贝壳找房" - def detailUrl = e.select(".title a").attr("href") - detail.url = detailUrl - def dbo = findTrade(detail) - if (!dbo) { - def houseInfo = e.select(".houseInfo").text() - def houseInfos = houseInfo.split("\\|") - detail.direction = houseInfos[0] - detail.fitment = houseInfos[1] - def floorStr = e.select(".positionInfo").text() - if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { - floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) - if (floorStr && floorStr.isNumber()) { - detail.height = floorStr.toInteger() - } - } - detail.create_date = new Date() - detail.m_date = new Date() - println detail - DEAL_PRICE_COLL.save(detail) + if(detail.total_price){ + def detailUrl = e.select(".title a").attr("href") + detail.url = detailUrl + def dbo = findTrade(detail) + if (!dbo) { + def houseInfo = e.select(".houseInfo").text() + def houseInfos = houseInfo.split("\\|") + detail.direction = houseInfos[0] + detail.fitment = houseInfos[1] + def floorStr = e.select(".positionInfo").text() + if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { + floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) + if (floorStr && floorStr.isNumber()) { + detail.height = floorStr.toInteger() + } + } + + detail.create_date = new Date() + detail.m_date = new Date() + println detail + DEAL_PRICE_COLL.save(detail) + } } } }catch(Exception ex){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 0ecdddeb1..e82b8bc45 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -328,25 +328,29 @@ public class LianjiaCrawlByChrome { } detail.source = "链家在线" detail.url = e.select(".title a").attr("href") - def dbo = findTrade(detail) - if (!dbo) { - def houseInfo = e.select(".houseInfo").text() - def houseInfos = houseInfo.split("\\|") - detail.direction = houseInfos[0] - detail.fitment = houseInfos[1] - def floorStr = e.select(".positionInfo").text() - if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { - floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) - if (floorStr && floorStr.isNumber()) { - detail.height = floorStr.toInteger() - } - } - detail.create_date = new Date() - detail.m_date = new Date() - println detail - DEAL_PRICE_COLL.save(detail) + if(detail.total_price){ + def dbo = findTrade(detail) + if (!dbo) { + def houseInfo = e.select(".houseInfo").text() + def houseInfos = houseInfo.split("\\|") + detail.direction = houseInfos[0] + detail.fitment = houseInfos[1] + def floorStr = e.select(".positionInfo").text() + if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { + floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) + if (floorStr && floorStr.isNumber()) { + detail.height = floorStr.toInteger() + } + } + + detail.create_date = new Date() + detail.m_date = new Date() + println detail + DEAL_PRICE_COLL.save(detail) + } } + } } }catch(Exception ex){