diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index af8723b42..eaf7586a5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -23,7 +23,7 @@ class ExportOracleData { def run() { // def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - + def new_dists = ["350604","350605","410307","410308","450181"] //不会变 exportEvaluationRateData() exportBigdataPoiData() @@ -37,9 +37,9 @@ class ExportOracleData { exportCommunityStaging() exportCommunityBuileType() - exportThirdPartyPrice2(lastMonth) - exportBigdataForsale("2022-04-19") - exportBigdataDeal("2022-04-01") + exportThirdPartyPrice2(lastMonth, new_dists) + exportBigdataForsale("2022-05-19", new_dists) + exportBigdataDeal("2022-05-01", new_dists) //计算结果相关 @@ -293,12 +293,51 @@ class ExportOracleData { println "COMMUNITY_GUIDE_PRICE_GJ 上传Ftp完成......" } - def exportBigdataDeal(def dealDate) { - def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + - "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + - "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + - "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + - "from bigdata_deal_gj where dealdate>=to_date('${dealDate}','yyyy-MM-dd')"; + def exportBigdataDeal(def dealDate, def new_dists) { + + def allRows = [] + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + + "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + + "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + + "from bigdata_deal_gj where dealdate>=to_date('${dealDate}','yyyy-MM-dd') and distid not in ${newstr}" + def rets1 = dealData(sql1) + + def sql2 = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + + "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + + "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + + "from bigdata_deal_gj where distid in ${newstr}" + def rets2 = dealData(sql2) + allRows.addAll(rets1) + allRows.addAll(rets2) + }else{ + def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + + "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + + "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + + "from bigdata_deal_gj where dealdate>=to_date('${dealDate}','yyyy-MM-dd')"; + allRows = dealData(sql) + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] +//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "BIGDATA_DEAL_GJ 数据文件输出完成......" + CcbFtpUtils.ftpUpload(null, fileName, curDate) + println "BIGDATA_DEAL_GJ 上传Ftp完成......" + } + + + def dealData(def sql){ def allRows = [] sqlInstance.eachRow(sql, { r -> def data = [] @@ -369,21 +408,9 @@ class ExportOracleData { data << r.WORKDATE allRows << data }); - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; - def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" - if (CommonDatautils.isWindow()) { - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" - } - def headers = [] -//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] - - FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); - println "BIGDATA_DEAL_GJ 数据文件输出完成......" - CcbFtpUtils.ftpUpload(null, fileName, curDate) - println "BIGDATA_DEAL_GJ 上传Ftp完成......" + return allRows } - def exportThirdPartyPrice() { def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, ZHUGE_AVG_PRICE, ANJUKE_AVG_PRICE, FANG_AVG_PRICE, LIANJIA_AVG_PRICE, CITYHOUSE_AVG_PRICE, MAITIAN_AVG_PRICE, KE_AVG_PRICE, \n" + "TAOFANG_AVG_PRICE, ZHONGYUAN_AVG_PRICE, TBS_AVG_PRICE, QFANG_AVG_PRICE, FANGTAN_AVG_PRICE, FANGDUODUO_AVG_PRICE, WIWJ_AVG_PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE\n" + @@ -433,8 +460,42 @@ class ExportOracleData { } - def exportThirdPartyPrice2(price_date) { - def sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where price_date=${price_date} "; + def exportThirdPartyPrice2(price_date, new_dists) { + + def allRows = [] + if(new_dists) { + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " + + "COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " + + "price_date=${price_date} and distid not in ${newstr} "; + def rets1 = getThirdPartyDatas(sql1) + allRows.addAll(rets1) + + def sql2 = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, " + + "COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where " + + "distid in ${newstr} "; + def rets2 = getThirdPartyDatas(sql2) + allRows.addAll(rets2) + }else{ + def sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where price_date=${price_date} "; + allRows = getThirdPartyDatas(sql) + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" +// fileName = "D:\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] +//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] + + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "BIGDATA_THIRD_PARTY_PRICE_GJ 数据文件输出完成......" + CcbFtpUtils.ftpUpload(null, fileName, curDate) + } + + def getThirdPartyDatas(def sql){ def allRows = [] sqlInstance.eachRow(sql, { r -> def data = [] @@ -452,21 +513,9 @@ class ExportOracleData { data << r.UPDATETIME allRows << data }) - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; - def fileName = "/ccb/MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" - if (CommonDatautils.isWindow()) { - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" -// fileName = "D:\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" - } - def headers = [] -//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] - - FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); - println "BIGDATA_THIRD_PARTY_PRICE_GJ 数据文件输出完成......" - CcbFtpUtils.ftpUpload(null, fileName, curDate) + return allRows; } - def exportBigdataCommunity(){ def sql = "select CITYID, DISTID, COMMID, DISTNAME, COMMNAME, ADDRESS, LATITUDE, LONGITUDE, LANDAREA, BUILDAREA, GREENINGRATE, PLOTRATIO, PARKINGCNT, BUILDYEAR, HOUSEUSE, ZTS, WORKDATE, UPDATEFLAG, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE, \n" + "\tCOMMALIAS, STREET, STREETNO, IFNEW, INTRODUCTION, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, SOURCECODE, XZJDB, LJXZQHDM, LJXZQH,STATUS from bigdata_community_gj ";//where cityId in (350200,150100) @@ -521,7 +570,7 @@ class ExportOracleData { println "BIGDATA_COMMUNITY_GJ 上传Ftp完成......" } - def exportBigdataForsale(publishDate){ + def exportBigdataForsale(publishDate, new_dists){ //["北京","上海"] def citys = CcbCommonUtils.citys; def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() @@ -535,84 +584,111 @@ class ExportOracleData { for(def city : citys){ def cityCode = cityCodeDataMap.get(city) - def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + - " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + - " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}' and publishtime>=to_date('${publishDate}','yyyy-MM-dd')"; - def allRows = [] - def index = 0 - sqlInstance.eachRow(sql,{r-> - index = index+1 - println city +" "+index - def data = [] - data << r.CITYID - data << r.FORSALEID - data << r.PROPCODE - data << r.CITYNAME - data << r.DISTID - data << r.DISTNAME - data << r.COMMID - data << r.COMMNAME - data << r.BUILDID - data << r.BUILDNO - data << r.BUILDNAME - data << r.UNITID - data << r.UNITNO - data << r.UNITNAME - data << r.HOUSEID - data << r.ADDRESS - data << r.BUILDYEAR - data << r.PROPTYPE - data << r.PROPYEARS - data << r.UNITPRICE - data << r.TOTALPRICE - data << r.PAYTYPE - data << r.FLOORNUM - data << r.FLOORCNT - data << r.BUILDTYPE - data << r.HOUSESTRU - data << r.BUILDSTRU - data << r.BUILDAREA - data << r.LIVINGAREA - data << r.HOUSESTYLE - data << r.ROOMCNT - data << r.LIVINGROOMCNT - data << r.KITCHENCNT - data << r.BATHROOMCNT - data << r.HOUSEASPECT - data << r.FITMENT - data << r.IFELEVATOR - data << r.ELEVATORHOUSE - data << r.PUBLISHTIME - data << r.EXPIRETIME - data << r.IFFIRST - data << r.DATASOURCE - data << r.AGENCYID - data << r.AGENCYNAME - data << r.PUBLISHER - data << r.PHONE - data << "" - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - data << r.ATTACH - allRows << data - if(allRows.size()%200000==0){ - FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); - allRows.clear() - } - }); + def allRows = [] + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj " + + "where cityId='${cityCode}' and publishtime>=to_date('${publishDate}','yyyy-MM-dd') and distid not in ${newstr}"; + def rets1 = getHouseDatas(sql1) + allRows.addAll(rets1) + }else{ + def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}' and publishtime>=to_date('${publishDate}','yyyy-MM-dd')"; + allRows = getHouseDatas(sql) + } if(allRows.size()>0){ FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); } + } + if(new_dists){//补充被修区域的数据 + def newstr = "('"+new_dists.join("','")+"')" + def sql2 = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH " + + "from bigdata_forsale_gj where distid in ${newstr}" + def rets2 = getHouseDatas(sql2) + if(rets2.size()>0){ + FileUtils.batchCreateFile(rets2, fileName, headers, "utf-8", true); + } } println "BIGDATA_FORSALE_GJ 数据文件输出完成......" CcbFtpUtils.ftpUpload(null, fileName, curDate) println "BIGDATA_FORSALE_GJ 上传Ftp完成......" } + def getHouseDatas(sql){ + def allRows = [] + def index = 0 + sqlInstance.eachRow(sql,{r-> + index = index+1 + println r.CITYNAME +" "+index + def data = [] + data << r.CITYID + data << r.FORSALEID + data << r.PROPCODE + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.UNITID + data << r.UNITNO + data << r.UNITNAME + data << r.HOUSEID + data << r.ADDRESS + data << r.BUILDYEAR + data << r.PROPTYPE + data << r.PROPYEARS + data << r.UNITPRICE + data << r.TOTALPRICE + data << r.PAYTYPE + data << r.FLOORNUM + data << r.FLOORCNT + data << r.BUILDTYPE + data << r.HOUSESTRU + data << r.BUILDSTRU + data << r.BUILDAREA + data << r.LIVINGAREA + data << r.HOUSESTYLE + data << r.ROOMCNT + data << r.LIVINGROOMCNT + data << r.KITCHENCNT + data << r.BATHROOMCNT + data << r.HOUSEASPECT + data << r.FITMENT + data << r.IFELEVATOR + data << r.ELEVATORHOUSE + data << r.PUBLISHTIME + data << r.EXPIRETIME + data << r.IFFIRST + data << r.DATASOURCE + data << r.AGENCYID + data << r.AGENCYNAME + data << r.PUBLISHER + data << r.PHONE + data << "" + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.ATTACH + allRows << data +// if(allRows.size()%200000==0){ +// FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); +// allRows.clear() +// } + }); + return allRows + } + def exportAvmEvaluationGJNB(yearMonth){ def sql = "select ID,CITYID,CITYNAME, YEAR_MONTH, COMMUNITY_NUM, HAS_PRICE_COMMUNITY_NUM, PRICE_COVERAGE, ARTIFICIAL_CALC_NUM, ARTIFICIAL_CALC_COVERAGE, SAMPLE_CALC_NUM, SAMPLE_CALC_COVERAGE, \n" + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 2476de4c3..045cd0a82 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2022-04-1", "2022-05-1") + def prices = CcbCommonUtils.findPersonPrice(city, "2022-05-1", "2022-06-1") if(prices){ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 5648796e8..72bc95878 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -334,6 +334,7 @@ ExcelUtils.modify("D:\\ if(dist){ def udata = new BasicDBObject() udata.put("name_cat", 11) + udata.put("status", 1) udata.put("m_date",new Date()) def q = new BasicDBObject() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy index 85e2c71b4..2ee14adc4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy @@ -31,8 +31,8 @@ class CcbDistrictToDB { // task.meituanSqrentRun() // task.updateDistrict() // task.updateDongguangRegion() -// task.houseDistToDB() - task.sendCalPrice() + task.houseDistToDB() +// task.sendCalPrice() } def sendCalPrice(){ @@ -97,11 +97,11 @@ class CcbDistrictToDB { def index = 0 def addIndex = 0 def keys = new CopyOnWriteArrayList() - ExcelUtils.modify("D:\\数据\\建行\\二期\\新增小区_2739_0610.xlsx", { d -> + ExcelUtils.modify("D:\\数据\\建行\\二期\\新增小区_2155.xlsx", { d -> index++ - if(d['I']){ + if(d['A']){ def city = d['A'] - def mapInfo = getStreetInfo(d['P']+","+d['Q']) + def mapInfo = getStreetInfo(d['L']+","+d['M']) def newRegion = null if(mapInfo){ if(d['A']=="东莞"){ @@ -114,15 +114,13 @@ class CcbDistrictToDB { newRegion = RegionData.formatRegion(city, newRegion) if(newRegion && newRegion.contains(d['B'])){ - def name = nameFileter(d['I']) + def name = nameFileter(d['E']) def alias = [] - if(d['I']!=name){ - alias << d['I'] - } - if(d['C']!=d['I'] && !alias.contains(d['C'])){ + if(d['C']!=name){ alias << d['C'] } + def estateType2 = "普通住宅" if(name){ if(name.contains("别墅")){ @@ -130,7 +128,7 @@ class CcbDistrictToDB { }else if(name.contains("商住")){ estateType2 = "商住" }else{ - def type = d['O'] + def type = d['K'] if(type){ if(type.contains("别墅")){ estateType2 = "别墅" @@ -141,8 +139,8 @@ class CcbDistrictToDB { } } def formatAddress = null - if (d['N']) { - formatAddress = AddressUtil.formatAddress(d['A'], d['N']) + if (d['J']) { + formatAddress = AddressUtil.formatAddress(d['A'], d['J']) } Long glat = null @@ -150,10 +148,10 @@ class CcbDistrictToDB { Long blat = null Long blng = null - if (d['P']) { - glat = d['P'].toString().toDouble() * 10000000000L - glng = d['Q'].toString().toDouble() * 10000000000L - def baidu = BaiDuApiUtils.geoconv(d['P'].toString().toDouble(), d['Q'].toString().toDouble(), 3) + if (d['M']) { + glat = d['M'].toString().toDouble() * 10000000000L + glng = d['L'].toString().toDouble() * 10000000000L + def baidu = BaiDuApiUtils.geoconv(d['L'].toString().toDouble(), d['M'].toString().toDouble(), 3) blng = baidu.x * 10000000000L blat = baidu.y * 10000000000L } @@ -175,6 +173,9 @@ class CcbDistrictToDB { data.put("name", name) data.put("estate_type2", estateType2) data.put("address", formatAddress) + if(alias){ + data.put("alias", alias) + } if (glat) { data.put("lat", glat) data.put("lng", glng) @@ -196,11 +197,47 @@ class CcbDistrictToDB { // } }else{ if(alias){ - def data = new BasicDBObject(); - data.put("alias", alias) - DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set',data)) + def dbAlias = dist.alias + if(!dbAlias){ + def data = new BasicDBObject(); + data.put("alias", alias) + DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set',data)) + }else{ + for(def n : alias){ + if(!dbAlias.contains(n)){ + dbAlias << n + } + } + + def data = new BasicDBObject(); + data.put("alias", dbAlias) + DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set',data)) + } } } + }else{ + if(alias){ + def dist = CommonDatautils.findDistrict(city, newRegion, name, "district") + if (dist) { + def dbAlias = dist.alias + if(!dbAlias){ + def data = new BasicDBObject(); + data.put("alias", alias) + DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set',data)) + }else{ + for(def n : alias){ + if(!dbAlias.contains(n)){ + dbAlias << n + } + } + + def data = new BasicDBObject(); + data.put("alias", dbAlias) + DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set',data)) + } + } + } + } }else{ println d['A']+" "+d['B']+" "+d['C'] +" 经纬度所在区:"+newRegion diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy index 6a50bd201..006b8a875 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy @@ -30,8 +30,32 @@ class PaimaiTest { def task = new PaimaiTest(); // task.run(); // task.stat() - task.updatePaimai() +// task.updatePaimai() // task.updateFieldPaimai() + task.statCity() + } + + def statCity(){ + def q = new BasicDBObject() + def keys = new BasicDBObject() + keys.put("city", 1) + def dbcursor = paimai43_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def set = new HashSet() + def index = 0 + for(def d : dbcursor){ + index++ + if(index%10000==0){ + print "." + } + if(!set.contains(d.city)){ + set << d.city + } + } + + println set.size() + for(def s : set){ + println s + } } def updateFieldPaimai(){