Merge branch 'script_gj' into 'master'

update

See merge request !556
This commit is contained in:
2023-02-27 15:28:47 +08:00
11 changed files with 224 additions and 44 deletions
@@ -45,6 +45,9 @@ class CcbCommonUtils {
public static def sqlInstance = null
public static def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver")
public static def getRunningCitys(def args){
def citys = []
@@ -573,6 +576,10 @@ class CcbCommonUtils {
public static def deleteGuidedPrice(def dealdate){
def sql = "delete from guided_price_gj where dealdate=to_date('${dealdate}','yyyy-mm-dd')"
sqlInstance.executeUpdate(sql)
def sql2 = "delete from base.t0831_guided_price_gj_h where gujia_drc_dt='${dealdate}'"
pgsqlInstance.executeUpdate(sql2)
}
}
@@ -7,7 +7,7 @@ import org.apache.commons.codec.digest.DigestUtils
class CcbFtpTestUtils {
static def ftpServer = "192.168.0.86"//45
static def ftpServer = "192.168.0.45"//45
static def ftpPort = 17495
static def ftpUser = "ccb"
static def ftpPassword = "ccb!@#"
@@ -21,6 +21,9 @@ class DealPriceDataProcess {
def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
static main(args) {
def task = new DealPriceDataProcess()
// task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据
@@ -372,12 +375,12 @@ class DealPriceDataProcess {
def deleteThirdPartyData(){
def citys = CcbCommonUtils.citys_all
def month = 202212
def month = 202301
def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def set = new HashSet()
for(def city : citys) {
def rets = findPeGteThirdParty(city, 202301)
def rets = findPeGteThirdParty2(city, 202302)
def total = 0
println city +" "+rets.size()
if(rets){
@@ -385,8 +388,12 @@ class DealPriceDataProcess {
}
for(def m : rets){
def cityId = cityCodeDatas.get(city)
def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
sqlInstance236.executeUpdate(sql)
// def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
// sqlInstance236.executeUpdate(sql)
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityId}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
pgsqlInstance.executeUpdate(sql)
}
}
println set.toString()
@@ -412,6 +419,19 @@ class DealPriceDataProcess {
return rets
}
def findPeGteThirdParty2(cityName, yearmonth){
def sql = "SELECT gujia_comm_id,gujiabds_to_mkprc_mny from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_nm='${cityName}' and gujia_mns_rto>25"
def rets = []
pgsqlInstance.eachRow(sql,{r->
def map = [:]
map.commid = r.gujia_comm_id
map.price = r.gujiabds_to_mkprc_mny
rets << map
});
return rets
}
def delDealData(){
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\成交\\杭州成交删除12-23.xlsx", { d ->
if(d['A']){
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2023-01-25", "2023-02-23")
def prices = CcbCommonUtils.findPersonPrice(city, "2023-01-25", "2023-02-27")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){
def distMap = getDistrictMap(city)
@@ -45,6 +45,9 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw
def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver")
//def citys = ["山南","黄南州","甘南","克孜勒苏","嘉峪关","海北州","阿拉善盟","怒江州","和田","日喀则","昌都","海南州","玉树州","吐鲁番","阿勒泰","三沙","塔城","甘孜州","果洛州","林芝","那曲","阿坝州","神农架"]
//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
@@ -1170,7 +1173,7 @@ def findThirdCommCount(cityid, sqlInstance){
//}
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\无价格小区(1).xlsx"
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\无价格小区(3)2.23.xlsx"
ExcelUtils.multiThreadingModify(fileName, { d ->
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
@@ -1180,6 +1183,48 @@ ExcelUtils.multiThreadingModify(fileName, { d ->
}
})
//def file = "C:\\Users\\gaojuan\\Desktop\\国信达小区.xlsx"
//def cityCodeMap = CcbCommonUtils.findCityCodeValue()
//ExcelUtils.multiThreadingModify(file, { d ->
// if(d['A']){
// def cityCode = cityCodeMap.get(d['A'])
// def newRegin = RegionData.formatRegion(d['A'], d['B'])
// if(newRegin){
// def dist = CommonDatautils.findDistrict(d['A'], newRegin, d['C'], "district")
// if(dist){
// d['H'] = dist._id.toString()
// def map = findAvgPrice(dist._id.toString(), pgsqlInstance, cityCode)
// if(map){
// d['I'] = map.avg_price
// d['J'] = map.calc_type
// d['K'] = map.month
// }
//
// }
// }
// }
//})
def findAvgPrice(def commid, def pgsqlInstance, def cityid){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
def map = [:]
pgsqlInstance.eachRow(sql, { r ->
map.avg_price = r.gujia_bds_clc_avprc
map.type = r.gujia_bds_clmtd
map.month = r.gujiabds_clc_yrmo_val
if(r.gujia_bds_clmtd == 1){
map.calc_type = "人工估值"
}else if(r.gujia_bds_clmtd == 2){
map.calc_type = "样本计算"
}else if(r.gujia_bds_clmtd == 3){
map.calc_type = "神经网络"
}
})
return map
}
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\0922入库数据.xlsx"
//def datas = ExcelUtils.read(fileName, 0, 1)
//def keys = []
@@ -33,24 +33,24 @@ class ExportPgsqlBaseEvaluation {
def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
def new_dists = []
// exportEvaluationRateData()
// exportBigdataPoiData()
// exportBigdataDistrictData()
exportEvaluationRateData()
exportBigdataPoiData()
exportBigdataDistrictData()
//
// //第三方网站权重表
// exportWebsiteRate();
//
// exportGuidedPrice()//随人工改变//
// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策//
// //处理小区情况
// exportCommunityStaging()//没cityid
// exportBigdataCommunity()//
//
// exportCommunityBuildType()//
//
// exportThirdPartyPrice2(lastMonth, new_dists)//
exportWebsiteRate();
exportGuidedPrice()//随人工改变//
exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策//
//处理小区情况
exportCommunityStaging()//没cityid
exportBigdataCommunity()//
exportCommunityBuildType()//
exportThirdPartyPrice2(lastMonth, new_dists)//
exportBigdataForsale(DateUtils.formatDateByPattern(new Date(), "yyyy-MM") +"-18", new_dists) //现在按updatetime导出
// exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01
exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01
}
def exportEvaluationRateData() {
@@ -34,22 +34,22 @@ class ExportPgsqlEvaluationResult {
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
//计算结果相关
// exportAvmEvaluationGJNB(curMonth)
// exportAVMEvaluationItemGJNB(curMonth)
exportAvmEvaluationGJNB(curMonth)
exportAVMEvaluationItemGJNB(curMonth)
exportCityPrice(curMonth)
exportRegionPrice(curMonth)
// exportCommunityBasePrice(curMonth)
exportCommunityBasePrice1(curMonth)
exportCommunityBasePrice(curMonth)
// exportCommunityBasePrice1(curMonth)
// exportCommunityPdescribeGj(curMonth)
// exportCommCaseDetailGj(curMonth)
exportCommunityPdescribeGj(curMonth)
exportCommCaseDetailGj(curMonth)
//
// exportInOk()
exportInOk()
//
// SSHHelperTest jschCommand = new SSHHelperTest();
// System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}"));
SSHHelperTest jschCommand = new SSHHelperTest();
System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}"));
}
@@ -162,7 +162,7 @@ class ExportPgsqlEvaluationResult {
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_city_price_gj_h where urbn_id='${cityCode}'";//gujiabds_clc_yrmo_val=${month} AND
def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_city_price_gj_h where gujiabds_clc_yrmo_val=${month} AND urbn_id='${cityCode}'";//
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_trdpt_avprc_id
@@ -189,7 +189,7 @@ class ExportPgsqlEvaluationResult {
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujia_admndiv_id,gxda_admndiv_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_district_price_gj_h where urbn_id='${cityCode}'"; //gujiabds_clc_yrmo_val=${month} AND
def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujia_admndiv_id,gxda_admndiv_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_district_price_gj_h where gujiabds_clc_yrmo_val=${month} AND urbn_id='${cityCode}'"; //
pgsqlInstance.eachRow(sql, { r ->
def data = []
@@ -220,7 +220,7 @@ class ExportPgsqlEvaluationResult {
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_bss_prc_id,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_bds_id,comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_comm_basic_prc,gujia_bds_clmtd,gujiabds_fwd_samp_rto,gujia_bds_mns_grd,gujia_bds_cfdrg,gujiabdsarndcmprccrlt,gujia_bds_cfdc_dgr,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_community_base_price_gj_h where urbn_id='${cityCode}'";//gujiabds_clc_yrmo_val=${yearMonth} AND
def sql = "select gujia_bss_prc_id,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_bds_id,comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_comm_basic_prc,gujia_bds_clmtd,gujiabds_fwd_samp_rto,gujia_bds_mns_grd,gujia_bds_cfdrg,gujiabdsarndcmprccrlt,gujia_bds_cfdc_dgr,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_community_base_price_gj_h where gujiabds_clc_yrmo_val=${yearMonth} AND urbn_id='${cityCode}'";//
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_bss_prc_id
@@ -74,7 +74,7 @@ class ExportDistrictRongHe extends BaseBatch{
exportBigdataModelingBase(lastMonth)
//2月8号 临时导出苏州一年的数据
// exportBigdataForsale("2022-07-01", new_dists) //现在按updatetime导出
exportBigdataForsale("2022-07-01", new_dists) //现在按updatetime导出
}
@@ -0,0 +1,56 @@
package com.fangjia.script.chongqing
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
class CzMatchDistrict {
static main(args) {
def task = new CzMatchDistrict();
task.run();
}
def run(){
// match("C:\\Users\\gaojuan\\Desktop\\有样本小区 .xlsx")
addFlag("C:\\Users\\gaojuan\\Desktop\\有样本小区 .xlsx")
}
def addFlag(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(row['B'] && row['G'] && row['B']==row['G']) {
// if(row['C'] && row['C'].contains("(")){
// def newValue = row['C'].replace("(","").replace(")","")
// row['J'] = newValue
// if(newValue == row['H'] && row['B'] == row['G']){
// row['K'] = "加别名1"
// }
// }
if (!row['K'] && row['C'].toString().endsWith("花园") && row['H']) {
def newValue = row['C'].replace("花园", "")
if (newValue && newValue == row['H']) {
row['K'] = "加别名3"
}
}
}
})
}
def match(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(row['C']){
def city = row['A']
def name = row['C']
def region = row['B']
println "[match] city:"+city+" region:"+region+" name:"+name
def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, name)
if(jsonObject != null && jsonObject.precise!=-1) {
row['G'] = jsonObject.get("district")
row['H'] = jsonObject.get("name")
row['I'] = jsonObject.get("id")
}
}
})
}
}
@@ -4,17 +4,24 @@ import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
class CityStatExport {
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", dbUrl)
public static final DBCollection HOUSE_LOUDONG = MongoUtils.getCollectionByName("fangjia", "house_loudong", "192.168.0.136")
public static final DBCollection LOUDONG = MongoUtils.getCollectionByName("fangjia_base", "loudong", "192.168.0.136")
static main(args) {
def task = new CityStatExport();
task.run()
@@ -25,16 +32,27 @@ class CityStatExport {
// def citys = CommonDatautils.findAllCity();
def citys = ["苏州","无锡","南通","徐州","连云港","宿迁","盐城","镇江","青岛","南京","常州","淮安","泰州","扬州","上海"]
def citys = ["太原","大同","临汾","朔州","忻州","晋中","阳泉","运城","长治","吕梁","晋城"]
def ret = []
// ret << ["城市","小区总量","产权年限覆盖率","竣工时间覆盖率"]
ret << ["城市","小区总量"]
ret << ["城市","楼栋总量", "房号总量"]
for(def city : citys){
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
def total = districts.size()
def m = []
m << city
m << total
// m << total
// def tradeCount = findTrades(city, '2022-02-01');
// m << tradeCount
// def houseCount = findHouseCount(city, '2022-02-01')
// m << houseCount
def loudongCount = findLoudongCount(city)
m << loudongCount
def fanghaoCount = findFanghaoCount(city)
m << fanghaoCount
// def priceCount = findDistrictPrice(city, null);
// m << priceCount
@@ -45,7 +63,7 @@ class CityStatExport {
ret << m
println m
}
ExcelUtils.write("D:\\数据\\小区数据统计.xlsx",ret,"数据")
ExcelUtils.write("D:\\数据\\山西省城市数据统计(楼栋房号).xlsx",ret,"数据")
}
@@ -62,6 +80,40 @@ class CityStatExport {
}
def findHouseCount(def city, def startDate){
def total = 0
try{
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} "
ElasticUtils.executeSqlQuery(sql,{ data->
total = data.c
})
}catch(Exception e){
e.printStackTrace()
}
return total
}
def findLoudongCount(String city) {
return LOUDONG.count(new BasicDBObject('city', city))
}
def findFanghaoCount(String city){
return HOUSE_LOUDONG.count(new BasicDBObject('city', city))
}
def findTrades(city, date){
def startDate = DateUtils.parseFromTightByPattern(date, "yyyy-MM-dd")
def q = new BasicDBObject()
q.put("city", city)
q.put("area", new BasicDBObject(QueryOperators.GTE,20))
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, startDate))
def total = DEAL_PRICE.count(q)
return total;
}
def findDistrictPrice(def city, def region){
def weekend = DateUtils.getFirstDayOfWeek(new Date());
DBObject q = new BasicDBObject()
@@ -136,8 +136,8 @@ class EsMonthHouseStat {
def houseRun(){
ExcelUtils.multiThreadingModify("d:\\数据\\建行\\城市覆盖率统计.xlsx",{d->
def city = d['A']
def count = findHouseCount(city, 20190709, 20210709)
d['E'] = count
def count = findHouseCount(city, 20221101, 20230201)
d['B'] = count
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"]);
def disthouseCount = 0
@@ -152,8 +152,8 @@ class EsMonthHouseStat {
alias << dist.name
}
def distCount = findHouseCount(dist.city, dist.region, alias, 20190709, 20210709)
if(distCount){
def distCount = findHouseCount(dist.city, dist.region, alias, 20221101, 20230201)
if(distCount && distCount>=3){
disthouseCount++
}
println dist.city+" "+dist.region+" "+dist.name+" "+distCount
@@ -162,7 +162,7 @@ class EsMonthHouseStat {
println "disthouseCount:"+disthouseCount
println districts.size()
def precent = DataHandleUtils.getDoubleRound(2, disthouseCount*100.0/districts.size())
d['F'] = precent+"%"
d['C'] = precent+"%"
},0)
}