Merge branch 'script_gj' into 'master'

融合修改,新增6张表

See merge request !506
This commit is contained in:
2023-01-09 17:21:38 +08:00
3 changed files with 267 additions and 6 deletions
@@ -4,10 +4,12 @@ import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.ccbevaluation.CcbFtpTestUtils
import com.fangjia.script.ccbevaluation.CcbFtpUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.jcraft.jsch.ChannelSftp
import groovy.sql.Sql
import java.sql.ResultSet
@@ -28,10 +30,11 @@ class ExportDistrictRongHe extends BaseBatch{
@Override
void process(Object args) {
citys = CcbCommonUtils.getRunningCitys(args)
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
def new_dists = []//["350604","350605","410307","410308","450181"]
//按天导(每天一个文件夹)
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists)
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists)
if(curDate.toString().endsWith("10")){
exportBigdataCommunityGJRH()//
@@ -51,8 +54,15 @@ class ExportDistrictRongHe extends BaseBatch{
exportEvaluateRate()
//因为融合暂不切库,所以需要加传一张第三方均价表到融合中, 从1月10日开始传输
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
exportThirdPartyPrice2(lastMonth, new_dists)
//1月10日 新增5张表,导92城,增量数据,2月10日导344城,其中252城所有历史数据,92城增量,3月后可能不用提供数据
exportCommunityBasePrice(lastMonth)
exportCommunityPdescribeGj(lastMonth)
exportGuideStatistics(lastMonth)
exportBigdataSupplierStatistics(lastMonth)
}
SSHHelper jschCommand = new SSHHelper();
System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
@@ -831,7 +841,12 @@ class ExportDistrictRongHe extends BaseBatch{
def rets2 = getThirdPartyDatas(sql2)
allRows.addAll(rets2)
}else{
def sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where CITYID='${cityCode}' and price_date=${price_date} ";
def sql = null
if(CcbCommonUtils.citys_92.contains(city)){
sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where CITYID='${cityCode}' and price_date=${price_date} ";
}else{
sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where CITYID='${cityCode}'";
}
allRows = getThirdPartyDatas(sql)
}
if(allRows && allRows.size()>0){
@@ -880,6 +895,252 @@ class ExportDistrictRongHe extends BaseBatch{
println "BIGDATA_BUILDPROJ_FJW 上传Ftp完成......"
}
def exportCommunityBasePrice(yearMonth){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = null;
if(CcbCommonUtils.citys_92.contains(city)){
sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth} AND CITYID='${cityCode}'";
}else{
sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where CITYID='${cityCode}'";
}
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.COMMID
data << r.COMMNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.AVG_PRICE
data << r.CALC_TYPE
data << r.FAKE_RATE
data << r.DIFF_RATE
data << r.CONFI_RATE
data << r.CORR_RATE
data << r.CONFIDENCE
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
}
def headers = []
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "COMMUNITY_BASE_PRICE_GJNB 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......"
}
def exportCommunityPdescribeGj(month){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = null;
if(CcbCommonUtils.citys_92.contains(city)){
sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month} AND CITYID='${cityCode}'";
}else{
sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where CITYID='${cityCode}'";
}
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.COMMUNITY_ID
data << r.CITYID
data << r.CITY_NAME
data << r.DISTRICT_CODE
data << r.DISTRICT_NAME
data << r.COMMUNITY_NAME
data << r.PQ_NAME
data << r.BASE_PRICE
data << r.CONFIDENCE_LEVEL
data << r.CNT
data << r.MIN_PRICE
data << r.MEDIANPRICE
data << r.MEANPRICE
data << r.MAX_PRICE
data << r.TIME
data << r.FIRSTDATAT
data << r.THIRDDATA
data << r.RELATIVEERROR
data << r.SKEWNESS
data << r.PQ_PRICE_RANGE
data << r.DISTRICT_PRICE_RANGE
data << r.DESCRIPTION
data << r.PRIV_BASE_PRICE
data << r.IS_ARTTFICIAL
data << r.PRIV_FINAL_PRICE
data << r.CREATE_DATE
data << r.PRICE
data << r.PRICE_SOURCE
data << r.CASE_MD_PRICE
data << r.CASE_AVG_PRICE
data << r.INDEX_LABEL
data << r.MAIN_PROPERTY
data << r.COMMTYPE
data << r.NEW_DEAL_PRICE
data << r.NEW_DEAL_DATE
data << r.PRICE_MOM_RATIO
data << r.MAX_MOM_RATIO
data << r.PRICE_YOY_RATIO
data << r.SATUR_RATIO
data << r.NEIGHBOR_PRICE
allRows << data
});
}
println allRows.size()
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "community_p_describe_gj 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
println "community_p_describe_gj 上传Ftp完成......"
}
def exportGuideStatistics(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
}
def headers = []
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = null;
if(CcbCommonUtils.citys_92.contains(city)){
sql = "select MODEL_TYPE,CITYID,CITYNAME,YEAR_MONTH,COMM_SUM_COUNT,COMM_COUNT,RATIO,TYPE from GUIDE_STATISTICS where cityid='${cityCode}' and year_month=${time}";
}else{
sql = "select MODEL_TYPE,CITYID,CITYNAME,YEAR_MONTH,COMM_SUM_COUNT,COMM_COUNT,RATIO,TYPE from GUIDE_STATISTICS where cityid='${cityCode}'";
}
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.MODEL_TYPE
data << r.CITYID
data << r.CITYNAME
data << r.YEAR_MONTH
data << r.COMM_SUM_COUNT
data << r.COMM_COUNT
data << r.RATIO
data << r.TYPE
allRows << data
});
}
println allRows.size()
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "guide_statistics 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
println "guide_statistics 上传Ftp完成......"
}
def exportBigdataSupplierStatistics(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
}
def headers = []
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = null;
if(CcbCommonUtils.citys_92.contains(city)){
sql = "select CITYID,CITYNAME,YEAR_MONTH,DATA_SOURCE,THIRD_COMM_COUNT,MODEL_TYPE from BIGDATA_SUPPLIER_STATISTICS where cityid='${cityCode}' and year_month=${time}";
}else{
sql = "select CITYID,CITYNAME,YEAR_MONTH,DATA_SOURCE,THIRD_COMM_COUNT,MODEL_TYPE from BIGDATA_SUPPLIER_STATISTICS where cityid='${cityCode}'";
}
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.CITYID
data << r.CITYNAME
data << r.YEAR_MONTH
data << r.DATA_SOURCE
data << r.THIRD_COMM_COUNT
data << r.MODEL_TYPE
allRows << data
});
}
println allRows.size()
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "bigdata_supplier_statistics 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
println "bigdata_supplier_statistics 上传Ftp完成......"
}
def exportBigdataModelingBase(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
}
def headers = []
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = null;
if(CcbCommonUtils.citys_92.contains(city)){
sql = "select CITY_CODE,CITYNAME,YEAR_MONTH,ESTIMATE_DEAL_COUNT,ESTIMATE_COMM_COUNT,DEAL_COUNT,DEAL_COMM_COUNT,SALE_COUNT,SALE_COMM_COUNT,MODEL_TYPE,TYPE from BIGDATA_MODELING_BASE where city_code='${cityCode}' and year_month=${time}";
}else{
sql = "select CITY_CODE,CITYNAME,YEAR_MONTH,ESTIMATE_DEAL_COUNT,ESTIMATE_COMM_COUNT,DEAL_COUNT,DEAL_COMM_COUNT,SALE_COUNT,SALE_COMM_COUNT,MODEL_TYPE,TYPE from BIGDATA_MODELING_BASE where city_code='${cityCode}'";
}
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.CITY_CODE
data << r.CITYNAME
data << r.YEAR_MONTH
data << r.ESTIMATE_DEAL_COUNT
data << r.ESTIMATE_COMM_COUNT
data << r.DEAL_COUNT
data << r.DEAL_COMM_COUNT
data << r.SALE_COUNT
data << r.SALE_COMM_COUNT
data << r.MODEL_TYPE
data << r.TYPE
allRows << data
});
}
println allRows.size()
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "bigdata_modeling_base 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
println "bigdata_modeling_base 上传Ftp完成......"
}
def exportBigdataCommlayoutpicFjw(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
@@ -26,7 +26,7 @@ class SynchDistrictPrice extends BaseBatch{
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田",
// "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士"
def esSources = ["吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士"]
def esSources = ["吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士"]
static main(args) {
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
for(def source :sources){
def count1 = findTotalCount(source, 202210);
def count2 = findTotalCount(source, 202211);
def count1 = findTotalCount(source, 202211);
def count2 = findTotalCount(source, 202212);
println source +" "+count1 +" "+count2
}
}