diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 9cb9003d2..aa12539b9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -57,7 +57,7 @@ class BigdataDealBatch extends BaseBatch{ // for(def city : citys){ // addDealPriceMatch(city); // } - println citys.unique().size() +// println citys.unique().size() def rows = [] def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 42f56e208..78153a5f0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -149,8 +149,8 @@ class CcbCommonUtils { public static def findTradesDetailById(dist, startDate, endDate){ def q = new BasicDBObject() - q.put("city", dist.city) - q.put("region", dist.region) + // q.put("city", dist.city) + // q.put("region", dist.region) q.put("area", new BasicDBObject('$gte', 20)) q.put("avg_price", new BasicDBObject('$gt', 0)) q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) @@ -161,7 +161,7 @@ class CcbCommonUtils { def lists = [] println dist.name +" "+total for(def d : dbcursor){ - def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd") + def key=d.city+"##"+d.match_region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd") if(!rets.contains(key)){ rets << key d.dist_id = dist._id.toString() @@ -367,7 +367,7 @@ class CcbCommonUtils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name","长隆小区") +// q.put("name","特房锦绣一方") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 1691b8656..20e49d87c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -21,14 +21,14 @@ class ExportOracleData { } def run() { - def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); +// def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); -// //不会变 + //不会变 exportEvaluationRateData() exportBigdataPoiData() exportBigdataDistrictData() -// + exportGuidedPrice()//随人工改变 exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy new file mode 100644 index 000000000..603a00f36 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy @@ -0,0 +1,415 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import groovy.sql.Sql +import com.fangjia.script.ccbevaluation.ronghe.SSHHelper + +class ExportOracleEvaluationData { + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + static main(args) { + def task = new ExportOracleEvaluationData() + task.run() + } + + def run() { + + def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); + + //计算结果相关 + exportAvmEvaluationGJNB(curMonth) + exportAVMEvaluationItemGJNB(curMonth) + + exportCityPrice(curMonth) + exportRegionPrice(curMonth) + + exportCommunityBasePriceGj(curMonth)//已合下面的合并 + exportCommunityBasePrice(curMonth) + + exportCommunityPdescribeGj(curMonth) + exportCommCaseDetailGj(curMonth) + + def curDate = curMonth.toString()+"28" + SSHHelper jschCommand = new SSHHelper(); + System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}")); + //ok文件 + exportOkFile() + } + + def exportAvmEvaluationGJNB(yearMonth){ + def sql = "select ID,CITYID,CITYNAME, YEAR_MONTH, COMMUNITY_NUM, HAS_PRICE_COMMUNITY_NUM, PRICE_COVERAGE, ARTIFICIAL_CALC_NUM, ARTIFICIAL_CALC_COVERAGE, SAMPLE_CALC_NUM, SAMPLE_CALC_COVERAGE, \n" + + "ANN_CALC_NUM, ANN_CALC_COVERAGE, TEST_NUM, TEST_COMM_NUM, TEST_COMM_COVERAGE, PPE15_COVERAGE, RATE_MEDIAN, COEFFICIENT,DEAL_TEST_CASE_NUM, DEAL_TEST_COMM_NUM, DEAL_TEST_COMM_COVERAGE, DEAL_PPE15_COVERAGE, \n" + + "DEAL_COEFFICIENT, DEAL_RATE_MEDIAN, THIRD_TEST_CASE_NUM, THIRD_TEST_COMM_NUM, THIRD_TEST_COMM_COVERAGE, THIRD_PPE15_COVERAGE, THIRD_COEFFICIENT, THIRD_RATE_MEDIAN, \n" + + "to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from avm_evaluation_gj where YEAR_MONTH = ${yearMonth}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.YEAR_MONTH + data << r.COMMUNITY_NUM + data << r.HAS_PRICE_COMMUNITY_NUM + data << r.PRICE_COVERAGE + data << r.ARTIFICIAL_CALC_NUM + data << r.ARTIFICIAL_CALC_COVERAGE + data << r.SAMPLE_CALC_NUM + data << r.SAMPLE_CALC_COVERAGE + data << r.ANN_CALC_NUM + data << r.ANN_CALC_COVERAGE + data << r.TEST_NUM + data << r.TEST_COMM_NUM + data << r.TEST_COMM_COVERAGE + data << r.PPE15_COVERAGE + data << r.RATE_MEDIAN + data << r.COEFFICIENT + data << r.DEAL_TEST_CASE_NUM + data << r.DEAL_TEST_COMM_NUM + data << r.DEAL_TEST_COMM_COVERAGE + data << r.DEAL_PPE15_COVERAGE + data << r.DEAL_COEFFICIENT + data << r.DEAL_RATE_MEDIAN + data << r.THIRD_TEST_CASE_NUM + data << r.THIRD_TEST_COMM_NUM + data << r.THIRD_TEST_COMM_COVERAGE + data << r.THIRD_PPE15_COVERAGE + data << r.THIRD_COEFFICIENT + data << r.THIRD_RATE_MEDIAN + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" + fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","YEAR_MONTH","COMMUNITY_NUM","HAS_PRICE_COMMUNITY_NUM","PRICE_COVERAGE","ARTIFICIAL_CALC_NUM","ARTIFICIAL_CALC_COVERAGE","SAMPLE_CALC_NUM","SAMPLE_CALC_COVERAGE", + //"ANN_CALC_NUM","ANN_CALC_COVERAGE","TEST_NUM","TEST_COMM_NUM","TEST_COMM_COVERAGE","PPE15_COVERAGE","RATE_MEDIAN","COEFFICIENT","DEAL_TEST_CASE_NUM","DEAL_TEST_COMM_NUM","DEAL_TEST_COMM_COVERAGE","DEAL_PPE15_COVERAGE", + //"DEAL_COEFFICIENT","DEAL_RATE_MEDIAN","THIRD_TEST_CASE_NUM","THIRD_TEST_COMM_NUM","THIRD_TEST_COMM_COVERAGE","THIRD_PPE15_COVERAGE","THIRD_COEFFICIENT","THIRD_RATE_MEDIAN","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false); + println "AVM_EVALUATION_GJNB 数据文件输出完成......" + CcbFtpUtils.ftpUpload(null, fileName, curDate) + CcbFtpUtils.ftpUpload(null, fileName1, curDate) + println "AVM_EVALUATION_GJNB 上传Ftp完成......" + } + + def exportAVMEvaluationItemGJNB(yearMonth){ + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, CONFIDENCE,TARGET_AVG_PRICE, \n" + + "to_char(DEALDATE,'yyyy-mm-dd hh24:mi:ss') as DEALDATE, UNIT_PRICE, BUILDAREA, FITMENT, HOUSEASPECT, FLOORNUM, FLOORCNT, DATA_TYPE, RATE, PE, \n" + + "to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME from avm_evaluation_item_gj where YEAR_MONTH = ${yearMonth}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.AVG_PRICE + data << r.CALC_TYPE + data << r.CONFIDENCE + data << r.TARGET_AVG_PRICE + data << r.DEALDATE + data << r.UNIT_PRICE + data << r.BUILDAREA + data << r.FITMENT + data << r.HOUSEASPECT + data << r.FLOORNUM + data << r.FLOORCNT + data << r.DATA_TYPE + data << r.RATE + data << r.PE + data << r.CREATETIME + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" + fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","CONFIDENCE","TARGET_AVG_PRICE","DEALDATE","UNIT_PRICE","BUILDAREA","FITMENT","HOUSEASPECT","FLOORNUM","FLOORCNT","DATA_TYPE","RATE","PE","CREATETIME"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false); + println "AVM_EVALUATION_ITEM_GJNB 数据文件输出完成......" + CcbFtpUtils.ftpUpload(null, fileName, curDate) + CcbFtpUtils.ftpUpload(null, fileName1, curDate) + println "AVM_EVALUATION_ITEM_GJNB 上传Ftp完成......" + } + + def exportCommunityBasePrice(yearMonth){ + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.AVG_PRICE + data << r.CALC_TYPE + data << r.FAKE_RATE + data << r.DIFF_RATE + data << r.CONFI_RATE + data << r.CORR_RATE + data << r.CONFIDENCE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false); + println "COMMUNITY_BASE_PRICE_GJNB 数据文件输出完成......" + CcbFtpUtils.ftpUpload(null, fileName, curDate) + CcbFtpUtils.ftpUpload(null, fileName1, curDate) + println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......" + } + + def exportCommunityBasePriceGj(yearMonth){ + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.AVG_PRICE + data << r.CALC_TYPE + data << r.FAKE_RATE + data << r.DIFF_RATE + data << r.CONFI_RATE + data << r.CORR_RATE + data << r.CONFIDENCE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "COMMUNITY_BASE_PRICE_GJ 数据文件输出完成......" + CcbFtpUtils.ftpUpload(null, fileName, curDate) + println "COMMUNITY_BASE_PRICE_GJ 上传Ftp完成......" + } + + def exportCityPrice(month){ + def sql = "select ID, CITYID, CITYNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from city_price_gj where year_month=${month}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.PRICE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def fileName = "/ccb/MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "CITY_PRICE_GJ 数据文件输出完成......" + CcbFtpUtils.ftpUpload(null, fileName, curDate) + println "CITY_PRICE_GJ 上传Ftp完成......" + } + + def exportRegionPrice(month){ + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from DISTRICT_PRICE_GJ where year_month=${month}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.PRICE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def fileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "DISTRICT_PRICE_GJ 数据文件输出完成......" + CcbFtpUtils.ftpUpload(null, fileName, curDate) + println "DISTRICT_PRICE_GJ 上传Ftp完成......" + } + + def exportCommunityPdescribeGj(month){ + +// def citys = CcbCommonUtils.citys; +// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] +// +// for(def city : citys) { +// def cityCode = cityCodeDataMap.get(city) + def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month}"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMUNITY_ID + data << r.CITYID + data << r.CITY_NAME + data << r.DISTRICT_CODE + data << r.DISTRICT_NAME + data << r.COMMUNITY_NAME + data << r.PQ_NAME + data << r.BASE_PRICE + data << r.CONFIDENCE_LEVEL + data << r.CNT + data << r.MIN_PRICE + data << r.MEDIANPRICE + data << r.MEANPRICE + data << r.MAX_PRICE + data << r.TIME + data << r.FIRSTDATAT + data << r.THIRDDATA + data << r.RELATIVEERROR + data << r.SKEWNESS + data << r.PQ_PRICE_RANGE + data << r.DISTRICT_PRICE_RANGE + data << r.DESCRIPTION + data << r.PRIV_BASE_PRICE + data << r.IS_ARTTFICIAL + data << r.PRIV_FINAL_PRICE + data << r.CREATE_DATE + data << r.PRICE + data << r.PRICE_SOURCE + data << r.CASE_MD_PRICE + data << r.CASE_AVG_PRICE + data << r.INDEX_LABEL + data << r.MAIN_PROPERTY + data << r.COMMTYPE + data << r.NEW_DEAL_PRICE + data << r.NEW_DEAL_DATE + data << r.PRICE_MOM_RATIO + data << r.MAX_MOM_RATIO + data << r.PRICE_YOY_RATIO + data << r.SATUR_RATIO + data << r.NEIGHBOR_PRICE + allRows << data + }); + println allRows.size() + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); +// } + println "community_p_describe_gj 数据文件输出完成......" + CcbFtpUtils.ftpUpload(null, fileName, curDate) + println "community_p_describe_gj 上传Ftp完成......" + } + + def exportCommCaseDetailGj(month){ + + def citys = CcbCommonUtils.citys; + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def fileName = "/ccb/MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] + + for(def city : citys){ + def cityCode = cityCodeDataMap.get(city) + def sql = "select ID, CITYID, COMMUNITY_ID, NETLISTING_ID, PRICE_DATE, to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE from comm_case_detail_gj where cityid='${cityCode}' and price_date=${month}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.COMMUNITY_ID + data << r.NETLISTING_ID + data << r.PRICE_DATE + data << r.CREATE_DATE + allRows << data + if(allRows.size()%200000==0){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + allRows.clear() + } + }); + if(allRows.size()>0){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + } + } + + println "comm_case_detail_gj 数据文件输出完成......" + CcbFtpUtils.ftpUpload(null, fileName, curDate) + println "comm_case_detail_gj 上传Ftp完成......" + } + + def exportOkFile(){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def dataCodeFileName = "/ccb/${curDate}.ok" + if(CommonDatautils.isWindow()){ + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok" + } + FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); + CcbFtpUtils.ftpUpload(null, dataCodeFileName, curDate) + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index dc639ef71..146c1f07b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -224,18 +224,18 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def } -//def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx" -//def dealMap = findDealCount(sqlInstance236, "2021-12-01", "2022-01-01") -//def testMap = findCaluateCityPPE(sqlInstance236, 202201) -//ExcelUtils.modify(fileName,{d-> -// if(d['C']){ -// def oldcount = dealMap.get(d['C']) -// d['T'] = oldcount -// def casecount = testMap.get(d['C']) -// d['U'] = casecount -// println d -// } -//},0) +def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx" +def dealMap = findDealCount(sqlInstance236, "2022-01-01", "2022-02-01") +def testMap = findCaluateCityPPE(sqlInstance236, 202202) +ExcelUtils.modify(fileName,{d-> + if(d['C']){ + def oldcount = dealMap.get(d['C']) + d['V'] = oldcount + def casecount = testMap.get(d['C']) + d['W'] = casecount + println d + } +},0) //ExcelUtils.multiThreadingModify("D:\\数据\\小区\\5个城市小区列表0723(1).xlsx",{d-> // if(d['I'] && d['K']){ @@ -270,24 +270,24 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def // } //},0) -def index = 0 -ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\320100_南京_是否独有及第三方是否存在(删除1049).xlsx",{d-> - def flag = d['M'] - if(flag==0.0){ - if(d['D']){ - def udata = new BasicDBObject() - udata.put("name_cat", 11) - udata.put("m_date",new Date()) - - def q = new BasicDBObject() - q.put("_id", new ObjectId(d['D'])) - DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) - index++ - println index - } - } - -},0) +//def index = 0 +//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\320100_南京_是否独有及第三方是否存在(删除1049).xlsx",{d-> +// def flag = d['M'] +// if(flag==0.0){ +// if(d['D']){ +// def udata = new BasicDBObject() +// udata.put("name_cat", 11) +// udata.put("m_date",new Date()) +// +// def q = new BasicDBObject() +// q.put("_id", new ObjectId(d['D'])) +// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +// index++ +// println index +// } +// } +// +//},0) //ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充删除数据1223_1228.xlsx",{d-> // if(d['A']){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index 3839acde9..621eac1ee 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -9,6 +9,7 @@ import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import com.mongodb.QueryOperators import groovy.sql.Sql import org.bson.types.ObjectId @@ -39,7 +40,7 @@ class TradeStatLog { static main(args) { def task = new TradeStatLog() - task.run()//统计最近3个月抓取的成交数据量 +// task.run()//统计最近3个月抓取的成交数据量 // task.exportTrade()//导出未入库成交 // task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) @@ -48,6 +49,7 @@ class TradeStatLog { // task.deleteDealData() // task.deleteDealData2() // task.statTableCount() + task.filterTrade() } @@ -479,8 +481,8 @@ class TradeStatLog { def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"] def rets = [] rets << headers - def startDate = "2021-12-01" - def endDate = "2022-01-01" + def startDate = "2022-01-01" + def endDate = "2022-02-01" for(def city : citys){ def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd")) @@ -561,7 +563,7 @@ class TradeStatLog { } println city +" "+rets.size() } - ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(12月).xlsx", rets,"数据") + ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(1月).xlsx", rets,"数据") } def deleteDistrict(){ @@ -706,4 +708,24 @@ class TradeStatLog { } + def filterTrade(){ + ExcelUtils.modify("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(1月).xlsx",{d-> + if(!d['J']){ + def count = findTradeById(d['A']) + if(count){ + d['J'] = "其它原因过滤" + } + } + },0) + } + + + def findTradeById(id){ + def q = new BasicDBObject() + q.put("_id", new ObjectId(id)) + q.put("match_id", new BasicDBObject(QueryOperators.EXISTS, true)) + def count = DEAL_PRICE_COLL.count(q) + return count + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MapAddressMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MapAddressMatch.groovy new file mode 100644 index 000000000..00efe7807 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MapAddressMatch.groovy @@ -0,0 +1,89 @@ +package com.fangjia.script.ccbevaluation.data + +import com.fangjia.core.RegionData +import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ExcelUtils + +class MapAddressMatch { + static main(args) { + def task = new MapAddressMatch() + task.run() + } + + def run(){ + def city = "北京" + def index = 0 + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\北京地址重复数据.xlsx", { d -> + index++ + println index +" "+ d['D']+" "+d['E'] + if(!d['G']){ +// def gdData = FangjiaClientUtils.getMapData(city, d['D'],d['E'], d['K'],"", "高德") +// if(gdData){ +// d['F'] = gdData.district +// d['G'] = gdData.name +// d['H'] = gdData.address +// d['I'] = gdData.type +// d['J'] = "高德" +// }else{ + def rets = OpenApiUtils.getGaoDeSearchLocation(city,d['D'],d['E']) + def map = findStandardObject(city,d['D'],d['E'], rets); + if (!map || !map.lat) { + rets = OpenApiUtils.getBiDuLocation(city,d['D'],d['E']); + map = findStandardObject(city,d['D'],d['E'], rets); + } + if(map){ + if(map.adname){ + d['F'] = map.adname + }else if(map.region){ + d['F'] = map.region + } + d['G'] = map.name + d['H'] = map.address + if(map.type){ + d['I'] = map.type + }else if(map.estateType){ + d['I'] = map.estateType + } + + d['J'] = map.source + } +// } + } + + },0) + } + + + + def findStandardObject(def city, def region, def name, def rets) { + def map; + if (rets) { + if (rets.size() >= 1) { + for (def mapObj : rets) { + if(mapObj.name && (mapObj.name.contains("(公交站)")||mapObj.name.contains("门)")||mapObj.name.contains("停车场"))){ + continue; + } +// if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){ + if(mapObj.name && (mapObj.name.contains(name) || name.contains(mapObj.name))){ + map = mapObj; + break; + }else if(mapObj.address && mapObj.address.contains(name)){ + map = mapObj; + break; + }else if(mapObj.name && name){ + double score1 = CosineSimilarAlgorithm.getSimilarity(mapObj.name,name); + if(score1){ + map = mapObj; + break; + } + } +// } + } + } + } + return map; + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ZsNoExistsToDB.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ZsNoExistsToDB.groovy index ce6949835..5c87fa8cb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ZsNoExistsToDB.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ZsNoExistsToDB.groovy @@ -134,7 +134,7 @@ class ZsNoExistsToDB { def run(){ def rets = findOpenLog2() - ExcelUtils.write("D:\\数据\\浙商\\不存在小区2.xlsx", rets, "数据") + // ExcelUtils.write("D:\\数据\\浙商\\不存在小区2.xlsx", rets, "数据") println rets.size() // matchMap2() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationUpdate.groovy index 0ad5dbfcf..90712ca06 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationUpdate.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationUpdate.groovy @@ -1,11 +1,16 @@ package com.fangjia.script.gj.location +import com.fangjia.core.util.GeoUtils import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.OpenApiUtils import com.fangjia.script.gj.district.NameProcess +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ExcelUtils import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.ResultSet - +import com.fangjia.script.gj.district.FangjiaClientUtils class DistrictLocationUpdate { public static EsDataSource dataSource = new EsDataSource() private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000" @@ -19,20 +24,177 @@ class DistrictLocationUpdate { static main(args) { def task = new DistrictLocationUpdate() - task.run() +// task.run() + task.mapMatch() } def run(){ - def citys = CommonDatautils.findAllCity() + def citys = ["福州"]//CommonDatautils.findAllCity() for(def city : citys){ - def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias", "lng", "lat"]) - - - updateLocation(city, dists); + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","address","alias", "lng", "lat"]) + def rets = [] + rets << ["城市","区域","名称","地址","经度","纬度","高德区域","高德名称","高德地址","高德类型","高德经度","高德纬度","高德类型","得分","距离"] + for(def d : dists){ + def lng = null + def lat = null + if(d.lng){ + lng = d.lng/10000000000.0 + lat = d.lat/10000000000.0 + } + rets << [d.city, d.region, d.name, d.address, lng,lat] + } + ExcelUtils.write("D:\\数据\\小区\\经纬度\\${city}.xlsx",rets, "数据") } } + def mapMatch(){ + def index = 0 + ExcelUtils.modify("D:\\数据\\小区\\经纬度\\福州V3.xlsx",{d-> + if(!d['H'] && d['C']=="凤凰小区"){ + index++ +// def gdData = FangjiaClientUtils.getMapData(d['A'], d['B'],d['C'], d['D'],"住宅", "高德") +// println d['A']+" "+d['B']+" "+d['C'] +// if(gdData){ +// d['G'] = gdData.district +// d['H'] = gdData.name +// d['I'] = gdData.address +// d['J'] = gdData.type +// d['K'] = gdData.lng +// d['L'] = gdData.lat +// d['M'] = gdData.score +// if(d['E'] && d['K']){ +// def distance = GeoUtils.calculateDistance(d['E'].toDouble(), d['F'].toDouble(), d['K'].toDouble(), d['L'].toDouble()) +// d['N'] = distance +// } +// } + + def rets = OpenApiUtils.getGaoDeSearchLocation(d['A'],d['B'],d['C']) + def map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets); + if (!map || !map.lat) { + rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']); + map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets); + } + if(map){ + if(map.adname){ + d['G'] = map.adname + }else if(map.region){ + d['G'] = map.region + } + d['H'] = map.name + d['I'] = map.address + if(map.type){ + d['J'] = map.type + }else if(map.estateType){ + d['J'] = map.estateType + } + d['K'] = map.lng + d['L'] = map.lat + d['M'] = map.score + if(d['E'] && d['K']){ + def distance = GeoUtils.calculateDistance(d['E'].toDouble(), d['F'].toDouble(), d['K'].toDouble(), d['L'].toDouble()) + d['N'] = distance + } + } + } + },0) + } + + + def findStandardObject2(def city, def region, def name, def address, def rets) { + if (rets) { + def bestObj = [:] + if (rets.size() >= 1) { + for (def mapObj : rets) { + def score = 0 + if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){ + score = 0.2 + } + if(mapObj.name && name){ + def newName = mapObj.name; + def names = [] + if(mapObj.name && mapObj.name.indexOf("(")>-1){ + def name1 = mapObj.name.substring(0, mapObj.name.indexOf("(")) + names << name1 + def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")")) + names << name2 + }else{ + names << newName + } + def maxNameScore = 0 + for(def n : names){ + double score1 = CosineSimilarAlgorithm.getSimilarity(n, name); + score1 = Eval.me(score1.toString()) + if(score1 && score1>0){ + def nameScore = (score1 * 4)/10 + if(nameScore > maxNameScore){ + maxNameScore = nameScore + } + } + } + + if(maxNameScore){ + maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore) + } + score = maxNameScore + score; + } + + if(mapObj.address && address){ + double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address); + if(addressScore && addressScore>0){ + addressScore = DataHandleUtils.getDoubleRound(2, addressScore) + score = addressScore*4/10 + score + } + } + + if(!bestObj){ + mapObj.score = score + bestObj = mapObj + }else{ + if(score > bestObj.score){ + mapObj.score = score + bestObj = mapObj + } + } + + } + } + return bestObj + } + return null + + } + + def findStandardObject(def city, def region, def name, def rets) { + def map; + if (rets) { + if (rets.size() >= 1) { + for (def mapObj : rets) { + if(mapObj.name && (mapObj.name.contains("(公交站)")||mapObj.name.contains("门)")||mapObj.name.contains("停车场"))){ + continue; + } +// if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){ + if(mapObj.name && (mapObj.name.contains(name) || name.contains(mapObj.name))){ + map = mapObj; + break; + }else if(mapObj.address && mapObj.address.contains(name)){ + map = mapObj; + break; + }else if(mapObj.name && name){ + double score1 = CosineSimilarAlgorithm.getSimilarity(mapObj.name,name); + if(score1){ + map = mapObj; + break; + } + } +// } + } + } + } + return map; + } + + def queryPoiMap(def city){ final StringBuffer sqlBuffer = new StringBuffer() sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi WHERE 1=1")