Merge branch 'script_gj' into 'master'

估值导出分2个文件

See merge request !262
This commit is contained in:
2022-02-24 16:22:18 +08:00
9 changed files with 738 additions and 50 deletions
@@ -57,7 +57,7 @@ class BigdataDealBatch extends BaseBatch{
// for(def city : citys){
// addDealPriceMatch(city);
// }
println citys.unique().size()
// println citys.unique().size()
def rows = []
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
@@ -149,8 +149,8 @@ class CcbCommonUtils {
public static def findTradesDetailById(dist, startDate, endDate){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
// q.put("city", dist.city)
// q.put("region", dist.region)
q.put("area", new BasicDBObject('$gte', 20))
q.put("avg_price", new BasicDBObject('$gt', 0))
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
@@ -161,7 +161,7 @@ class CcbCommonUtils {
def lists = []
println dist.name +" "+total
for(def d : dbcursor){
def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
def key=d.city+"##"+d.match_region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
if(!rets.contains(key)){
rets << key
d.dist_id = dist._id.toString()
@@ -367,7 +367,7 @@ class CcbCommonUtils {
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
// q.put("name","长隆小区")
// q.put("name","特房锦绣一方")
// q.put("name_cat", 0);
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
@@ -21,14 +21,14 @@ class ExportOracleData {
}
def run() {
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
// def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
// //不会变
//不会变
exportEvaluationRateData()
exportBigdataPoiData()
exportBigdataDistrictData()
//
exportGuidedPrice()//随人工改变
exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策
@@ -0,0 +1,415 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import groovy.sql.Sql
import com.fangjia.script.ccbevaluation.ronghe.SSHHelper
class ExportOracleEvaluationData {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new ExportOracleEvaluationData()
task.run()
}
def run() {
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
//计算结果相关
exportAvmEvaluationGJNB(curMonth)
exportAVMEvaluationItemGJNB(curMonth)
exportCityPrice(curMonth)
exportRegionPrice(curMonth)
exportCommunityBasePriceGj(curMonth)//已合下面的合并
exportCommunityBasePrice(curMonth)
exportCommunityPdescribeGj(curMonth)
exportCommCaseDetailGj(curMonth)
def curDate = curMonth.toString()+"28"
SSHHelper jschCommand = new SSHHelper();
System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
//ok文件
exportOkFile()
}
def exportAvmEvaluationGJNB(yearMonth){
def sql = "select ID,CITYID,CITYNAME, YEAR_MONTH, COMMUNITY_NUM, HAS_PRICE_COMMUNITY_NUM, PRICE_COVERAGE, ARTIFICIAL_CALC_NUM, ARTIFICIAL_CALC_COVERAGE, SAMPLE_CALC_NUM, SAMPLE_CALC_COVERAGE, \n" +
"ANN_CALC_NUM, ANN_CALC_COVERAGE, TEST_NUM, TEST_COMM_NUM, TEST_COMM_COVERAGE, PPE15_COVERAGE, RATE_MEDIAN, COEFFICIENT,DEAL_TEST_CASE_NUM, DEAL_TEST_COMM_NUM, DEAL_TEST_COMM_COVERAGE, DEAL_PPE15_COVERAGE, \n" +
"DEAL_COEFFICIENT, DEAL_RATE_MEDIAN, THIRD_TEST_CASE_NUM, THIRD_TEST_COMM_NUM, THIRD_TEST_COMM_COVERAGE, THIRD_PPE15_COVERAGE, THIRD_COEFFICIENT, THIRD_RATE_MEDIAN, \n" +
"to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from avm_evaluation_gj where YEAR_MONTH = ${yearMonth}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.YEAR_MONTH
data << r.COMMUNITY_NUM
data << r.HAS_PRICE_COMMUNITY_NUM
data << r.PRICE_COVERAGE
data << r.ARTIFICIAL_CALC_NUM
data << r.ARTIFICIAL_CALC_COVERAGE
data << r.SAMPLE_CALC_NUM
data << r.SAMPLE_CALC_COVERAGE
data << r.ANN_CALC_NUM
data << r.ANN_CALC_COVERAGE
data << r.TEST_NUM
data << r.TEST_COMM_NUM
data << r.TEST_COMM_COVERAGE
data << r.PPE15_COVERAGE
data << r.RATE_MEDIAN
data << r.COEFFICIENT
data << r.DEAL_TEST_CASE_NUM
data << r.DEAL_TEST_COMM_NUM
data << r.DEAL_TEST_COMM_COVERAGE
data << r.DEAL_PPE15_COVERAGE
data << r.DEAL_COEFFICIENT
data << r.DEAL_RATE_MEDIAN
data << r.THIRD_TEST_CASE_NUM
data << r.THIRD_TEST_COMM_NUM
data << r.THIRD_TEST_COMM_COVERAGE
data << r.THIRD_PPE15_COVERAGE
data << r.THIRD_COEFFICIENT
data << r.THIRD_RATE_MEDIAN
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat"
def fileName1 = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat"
fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat"
}
def headers = []//["ID","CITYID","CITYNAME","YEAR_MONTH","COMMUNITY_NUM","HAS_PRICE_COMMUNITY_NUM","PRICE_COVERAGE","ARTIFICIAL_CALC_NUM","ARTIFICIAL_CALC_COVERAGE","SAMPLE_CALC_NUM","SAMPLE_CALC_COVERAGE",
//"ANN_CALC_NUM","ANN_CALC_COVERAGE","TEST_NUM","TEST_COMM_NUM","TEST_COMM_COVERAGE","PPE15_COVERAGE","RATE_MEDIAN","COEFFICIENT","DEAL_TEST_CASE_NUM","DEAL_TEST_COMM_NUM","DEAL_TEST_COMM_COVERAGE","DEAL_PPE15_COVERAGE",
//"DEAL_COEFFICIENT","DEAL_RATE_MEDIAN","THIRD_TEST_CASE_NUM","THIRD_TEST_COMM_NUM","THIRD_TEST_COMM_COVERAGE","THIRD_PPE15_COVERAGE","THIRD_COEFFICIENT","THIRD_RATE_MEDIAN","CREATETIME","UPDATETIME","SOURCECODE"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false);
println "AVM_EVALUATION_GJNB 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
CcbFtpUtils.ftpUpload(null, fileName1, curDate)
println "AVM_EVALUATION_GJNB 上传Ftp完成......"
}
def exportAVMEvaluationItemGJNB(yearMonth){
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, CONFIDENCE,TARGET_AVG_PRICE, \n" +
"to_char(DEALDATE,'yyyy-mm-dd hh24:mi:ss') as DEALDATE, UNIT_PRICE, BUILDAREA, FITMENT, HOUSEASPECT, FLOORNUM, FLOORCNT, DATA_TYPE, RATE, PE, \n" +
"to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME from avm_evaluation_item_gj where YEAR_MONTH = ${yearMonth}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.COMMID
data << r.COMMNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.AVG_PRICE
data << r.CALC_TYPE
data << r.CONFIDENCE
data << r.TARGET_AVG_PRICE
data << r.DEALDATE
data << r.UNIT_PRICE
data << r.BUILDAREA
data << r.FITMENT
data << r.HOUSEASPECT
data << r.FLOORNUM
data << r.FLOORCNT
data << r.DATA_TYPE
data << r.RATE
data << r.PE
data << r.CREATETIME
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat"
def fileName1 = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat"
fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat"
}
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","CONFIDENCE","TARGET_AVG_PRICE","DEALDATE","UNIT_PRICE","BUILDAREA","FITMENT","HOUSEASPECT","FLOORNUM","FLOORCNT","DATA_TYPE","RATE","PE","CREATETIME"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false);
println "AVM_EVALUATION_ITEM_GJNB 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
CcbFtpUtils.ftpUpload(null, fileName1, curDate)
println "AVM_EVALUATION_ITEM_GJNB 上传Ftp完成......"
}
def exportCommunityBasePrice(yearMonth){
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.COMMID
data << r.COMMNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.AVG_PRICE
data << r.CALC_TYPE
data << r.FAKE_RATE
data << r.DIFF_RATE
data << r.CONFI_RATE
data << r.CORR_RATE
data << r.CONFIDENCE
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
def fileName1 = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false);
println "COMMUNITY_BASE_PRICE_GJNB 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
CcbFtpUtils.ftpUpload(null, fileName1, curDate)
println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......"
}
def exportCommunityBasePriceGj(yearMonth){
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.COMMID
data << r.COMMNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.AVG_PRICE
data << r.CALC_TYPE
data << r.FAKE_RATE
data << r.DIFF_RATE
data << r.CONFI_RATE
data << r.CORR_RATE
data << r.CONFIDENCE
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "COMMUNITY_BASE_PRICE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
println "COMMUNITY_BASE_PRICE_GJ 上传Ftp完成......"
}
def exportCityPrice(month){
def sql = "select ID, CITYID, CITYNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from city_price_gj where year_month=${month}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.PRICE
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb/MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat"
}
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "CITY_PRICE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
println "CITY_PRICE_GJ 上传Ftp完成......"
}
def exportRegionPrice(month){
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from DISTRICT_PRICE_GJ where year_month=${month}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.PRICE
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat"
}
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "DISTRICT_PRICE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
println "DISTRICT_PRICE_GJ 上传Ftp完成......"
}
def exportCommunityPdescribeGj(month){
// def citys = CcbCommonUtils.citys;
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []
//
// for(def city : citys) {
// def cityCode = cityCodeDataMap.get(city)
def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month}";
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.COMMUNITY_ID
data << r.CITYID
data << r.CITY_NAME
data << r.DISTRICT_CODE
data << r.DISTRICT_NAME
data << r.COMMUNITY_NAME
data << r.PQ_NAME
data << r.BASE_PRICE
data << r.CONFIDENCE_LEVEL
data << r.CNT
data << r.MIN_PRICE
data << r.MEDIANPRICE
data << r.MEANPRICE
data << r.MAX_PRICE
data << r.TIME
data << r.FIRSTDATAT
data << r.THIRDDATA
data << r.RELATIVEERROR
data << r.SKEWNESS
data << r.PQ_PRICE_RANGE
data << r.DISTRICT_PRICE_RANGE
data << r.DESCRIPTION
data << r.PRIV_BASE_PRICE
data << r.IS_ARTTFICIAL
data << r.PRIV_FINAL_PRICE
data << r.CREATE_DATE
data << r.PRICE
data << r.PRICE_SOURCE
data << r.CASE_MD_PRICE
data << r.CASE_AVG_PRICE
data << r.INDEX_LABEL
data << r.MAIN_PROPERTY
data << r.COMMTYPE
data << r.NEW_DEAL_PRICE
data << r.NEW_DEAL_DATE
data << r.PRICE_MOM_RATIO
data << r.MAX_MOM_RATIO
data << r.PRICE_YOY_RATIO
data << r.SATUR_RATIO
data << r.NEIGHBOR_PRICE
allRows << data
});
println allRows.size()
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
// }
println "community_p_describe_gj 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
println "community_p_describe_gj 上传Ftp完成......"
}
def exportCommCaseDetailGj(month){
def citys = CcbCommonUtils.citys;
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb/MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat"
}
def headers = []
for(def city : citys){
def cityCode = cityCodeDataMap.get(city)
def sql = "select ID, CITYID, COMMUNITY_ID, NETLISTING_ID, PRICE_DATE, to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE from comm_case_detail_gj where cityid='${cityCode}' and price_date=${month}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.COMMUNITY_ID
data << r.NETLISTING_ID
data << r.PRICE_DATE
data << r.CREATE_DATE
allRows << data
if(allRows.size()%200000==0){
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
allRows.clear()
}
});
if(allRows.size()>0){
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
}
}
println "comm_case_detail_gj 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
println "comm_case_detail_gj 上传Ftp完成......"
}
def exportOkFile(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def dataCodeFileName = "/ccb/${curDate}.ok"
if(CommonDatautils.isWindow()){
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok"
}
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload(null, dataCodeFileName, curDate)
}
}
@@ -224,18 +224,18 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
}
//def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx"
//def dealMap = findDealCount(sqlInstance236, "2021-12-01", "2022-01-01")
//def testMap = findCaluateCityPPE(sqlInstance236, 202201)
//ExcelUtils.modify(fileName,{d->
// if(d['C']){
// def oldcount = dealMap.get(d['C'])
// d['T'] = oldcount
// def casecount = testMap.get(d['C'])
// d['U'] = casecount
// println d
// }
//},0)
def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx"
def dealMap = findDealCount(sqlInstance236, "2022-01-01", "2022-02-01")
def testMap = findCaluateCityPPE(sqlInstance236, 202202)
ExcelUtils.modify(fileName,{d->
if(d['C']){
def oldcount = dealMap.get(d['C'])
d['V'] = oldcount
def casecount = testMap.get(d['C'])
d['W'] = casecount
println d
}
},0)
//ExcelUtils.multiThreadingModify("D:\\数据\\小区\\5个城市小区列表0723(1).xlsx",{d->
// if(d['I'] && d['K']){
@@ -270,24 +270,24 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
// }
//},0)
def index = 0
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\320100_南京_是否独有及第三方是否存在(删除1049).xlsx",{d->
def flag = d['M']
if(flag==0.0){
if(d['D']){
def udata = new BasicDBObject()
udata.put("name_cat", 11)
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", new ObjectId(d['D']))
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
index++
println index
}
}
},0)
//def index = 0
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\320100_南京_是否独有及第三方是否存在(删除1049).xlsx",{d->
// def flag = d['M']
// if(flag==0.0){
// if(d['D']){
// def udata = new BasicDBObject()
// udata.put("name_cat", 11)
// udata.put("m_date",new Date())
//
// def q = new BasicDBObject()
// q.put("_id", new ObjectId(d['D']))
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
// index++
// println index
// }
// }
//
//},0)
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充删除数据1223_1228.xlsx",{d->
// if(d['A']){
@@ -9,6 +9,7 @@ import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.QueryOperators
import groovy.sql.Sql
import org.bson.types.ObjectId
@@ -39,7 +40,7 @@ class TradeStatLog {
static main(args) {
def task = new TradeStatLog()
task.run()//统计最近3个月抓取的成交数据量
// task.run()//统计最近3个月抓取的成交数据量
// task.exportTrade()//导出未入库成交
// task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
@@ -48,6 +49,7 @@ class TradeStatLog {
// task.deleteDealData()
// task.deleteDealData2()
// task.statTableCount()
task.filterTrade()
}
@@ -479,8 +481,8 @@ class TradeStatLog {
def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"]
def rets = []
rets << headers
def startDate = "2021-12-01"
def endDate = "2022-01-01"
def startDate = "2022-01-01"
def endDate = "2022-02-01"
for(def city : citys){
def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
@@ -561,7 +563,7 @@ class TradeStatLog {
}
println city +" "+rets.size()
}
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(12月).xlsx", rets,"数据")
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(1月).xlsx", rets,"数据")
}
def deleteDistrict(){
@@ -706,4 +708,24 @@ class TradeStatLog {
}
def filterTrade(){
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(1月).xlsx",{d->
if(!d['J']){
def count = findTradeById(d['A'])
if(count){
d['J'] = "其它原因过滤"
}
}
},0)
}
def findTradeById(id){
def q = new BasicDBObject()
q.put("_id", new ObjectId(id))
q.put("match_id", new BasicDBObject(QueryOperators.EXISTS, true))
def count = DEAL_PRICE_COLL.count(q)
return count
}
}
@@ -0,0 +1,89 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.core.RegionData
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils
class MapAddressMatch {
static main(args) {
def task = new MapAddressMatch()
task.run()
}
def run(){
def city = "北京"
def index = 0
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\北京地址重复数据.xlsx", { d ->
index++
println index +" "+ d['D']+" "+d['E']
if(!d['G']){
// def gdData = FangjiaClientUtils.getMapData(city, d['D'],d['E'], d['K'],"", "高德")
// if(gdData){
// d['F'] = gdData.district
// d['G'] = gdData.name
// d['H'] = gdData.address
// d['I'] = gdData.type
// d['J'] = "高德"
// }else{
def rets = OpenApiUtils.getGaoDeSearchLocation(city,d['D'],d['E'])
def map = findStandardObject(city,d['D'],d['E'], rets);
if (!map || !map.lat) {
rets = OpenApiUtils.getBiDuLocation(city,d['D'],d['E']);
map = findStandardObject(city,d['D'],d['E'], rets);
}
if(map){
if(map.adname){
d['F'] = map.adname
}else if(map.region){
d['F'] = map.region
}
d['G'] = map.name
d['H'] = map.address
if(map.type){
d['I'] = map.type
}else if(map.estateType){
d['I'] = map.estateType
}
d['J'] = map.source
}
// }
}
},0)
}
def findStandardObject(def city, def region, def name, def rets) {
def map;
if (rets) {
if (rets.size() >= 1) {
for (def mapObj : rets) {
if(mapObj.name && (mapObj.name.contains("(公交站)")||mapObj.name.contains("门)")||mapObj.name.contains("停车场"))){
continue;
}
// if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
if(mapObj.name && (mapObj.name.contains(name) || name.contains(mapObj.name))){
map = mapObj;
break;
}else if(mapObj.address && mapObj.address.contains(name)){
map = mapObj;
break;
}else if(mapObj.name && name){
double score1 = CosineSimilarAlgorithm.getSimilarity(mapObj.name,name);
if(score1){
map = mapObj;
break;
}
}
// }
}
}
}
return map;
}
}
@@ -134,7 +134,7 @@ class ZsNoExistsToDB {
def run(){
def rets = findOpenLog2()
ExcelUtils.write("D:\\数据\\浙商\\不存在小区2.xlsx", rets, "数据")
// ExcelUtils.write("D:\\数据\\浙商\\不存在小区2.xlsx", rets, "数据")
println rets.size()
// matchMap2()
@@ -1,11 +1,16 @@
package com.fangjia.script.gj.location
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.gj.district.NameProcess
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import com.fangjia.script.gj.district.FangjiaClientUtils
class DistrictLocationUpdate {
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
@@ -19,20 +24,177 @@ class DistrictLocationUpdate {
static main(args) {
def task = new DistrictLocationUpdate()
task.run()
// task.run()
task.mapMatch()
}
def run(){
def citys = CommonDatautils.findAllCity()
def citys = ["福州"]//CommonDatautils.findAllCity()
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias", "lng", "lat"])
updateLocation(city, dists);
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","address","alias", "lng", "lat"])
def rets = []
rets << ["城市","区域","名称","地址","经度","纬度","高德区域","高德名称","高德地址","高德类型","高德经度","高德纬度","高德类型","得分","距离"]
for(def d : dists){
def lng = null
def lat = null
if(d.lng){
lng = d.lng/10000000000.0
lat = d.lat/10000000000.0
}
rets << [d.city, d.region, d.name, d.address, lng,lat]
}
ExcelUtils.write("D:\\数据\\小区\\经纬度\\${city}.xlsx",rets, "数据")
}
}
def mapMatch(){
def index = 0
ExcelUtils.modify("D:\\数据\\小区\\经纬度\\福州V3.xlsx",{d->
if(!d['H'] && d['C']=="凤凰小区"){
index++
// def gdData = FangjiaClientUtils.getMapData(d['A'], d['B'],d['C'], d['D'],"住宅", "高德")
// println d['A']+" "+d['B']+" "+d['C']
// if(gdData){
// d['G'] = gdData.district
// d['H'] = gdData.name
// d['I'] = gdData.address
// d['J'] = gdData.type
// d['K'] = gdData.lng
// d['L'] = gdData.lat
// d['M'] = gdData.score
// if(d['E'] && d['K']){
// def distance = GeoUtils.calculateDistance(d['E'].toDouble(), d['F'].toDouble(), d['K'].toDouble(), d['L'].toDouble())
// d['N'] = distance
// }
// }
def rets = OpenApiUtils.getGaoDeSearchLocation(d['A'],d['B'],d['C'])
def map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets);
if (!map || !map.lat) {
rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']);
map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets);
}
if(map){
if(map.adname){
d['G'] = map.adname
}else if(map.region){
d['G'] = map.region
}
d['H'] = map.name
d['I'] = map.address
if(map.type){
d['J'] = map.type
}else if(map.estateType){
d['J'] = map.estateType
}
d['K'] = map.lng
d['L'] = map.lat
d['M'] = map.score
if(d['E'] && d['K']){
def distance = GeoUtils.calculateDistance(d['E'].toDouble(), d['F'].toDouble(), d['K'].toDouble(), d['L'].toDouble())
d['N'] = distance
}
}
}
},0)
}
def findStandardObject2(def city, def region, def name, def address, def rets) {
if (rets) {
def bestObj = [:]
if (rets.size() >= 1) {
for (def mapObj : rets) {
def score = 0
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
score = 0.2
}
if(mapObj.name && name){
def newName = mapObj.name;
def names = []
if(mapObj.name && mapObj.name.indexOf("(")>-1){
def name1 = mapObj.name.substring(0, mapObj.name.indexOf("("))
names << name1
def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")"))
names << name2
}else{
names << newName
}
def maxNameScore = 0
for(def n : names){
double score1 = CosineSimilarAlgorithm.getSimilarity(n, name);
score1 = Eval.me(score1.toString())
if(score1 && score1>0){
def nameScore = (score1 * 4)/10
if(nameScore > maxNameScore){
maxNameScore = nameScore
}
}
}
if(maxNameScore){
maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore)
}
score = maxNameScore + score;
}
if(mapObj.address && address){
double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address);
if(addressScore && addressScore>0){
addressScore = DataHandleUtils.getDoubleRound(2, addressScore)
score = addressScore*4/10 + score
}
}
if(!bestObj){
mapObj.score = score
bestObj = mapObj
}else{
if(score > bestObj.score){
mapObj.score = score
bestObj = mapObj
}
}
}
}
return bestObj
}
return null
}
def findStandardObject(def city, def region, def name, def rets) {
def map;
if (rets) {
if (rets.size() >= 1) {
for (def mapObj : rets) {
if(mapObj.name && (mapObj.name.contains("(公交站)")||mapObj.name.contains("门)")||mapObj.name.contains("停车场"))){
continue;
}
// if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
if(mapObj.name && (mapObj.name.contains(name) || name.contains(mapObj.name))){
map = mapObj;
break;
}else if(mapObj.address && mapObj.address.contains(name)){
map = mapObj;
break;
}else if(mapObj.name && name){
double score1 = CosineSimilarAlgorithm.getSimilarity(mapObj.name,name);
if(score1){
map = mapObj;
break;
}
}
// }
}
}
}
return map;
}
def queryPoiMap(def city){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi WHERE 1=1")