Merge branch 'script_gj' into 'master'

update

See merge request !760
This commit is contained in:
2023-11-30 10:45:26 +08:00
8 changed files with 183 additions and 66 deletions
@@ -28,8 +28,8 @@ class DealPriceDataProcess {
def task = new DealPriceDataProcess()
// task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据
// task.addLianJiaPrice();
task.deleteThirdPartyData()//删除第三方
// task.delDealByPe()
// task.deleteThirdPartyData()//删除第三方
task.delDealByPe()
// task.delDealData()
// task.exportThirdParty()
@@ -37,49 +37,50 @@ class DealPriceDataProcess {
}
def delDealByPe(){
def yearMonth = 202201
def yearMonth = 202311
def thirdMonth = 202112
// def rets = findAllDealByPe(yearMonth, 70)
// for(def r : rets){
// deleteDealPrice(r)
def rets = findAllDealByPe(yearMonth, 18)
for(def r : rets){
deleteDealPrice(r)
}
// def list = findAllThirdByPe(yearMonth, 70)
// for(def r : list){
// deletThirdPrice(r, thirdMonth)
// }
// def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\估值模型测试202201_1221 - 副本.xlsx",0,1)
// for(def d : datas){
//// if(d['G']){
//// def rets = findDealByPe(d['A'], yearMonth, d['G'])
//// for(def r : rets){
//// deleteDealPrice(r)
//// }
//// }
//
//// if(d['H']){
//// def rets = findCityByPpeRange(d['A'], yearMonth, 15, 20)
//// for(def m : rets){
//// def precent = d['H']/100.0
//// def newPrice = null
//// if(m.unit_price > m.deal_price){
//// newPrice = m.deal_price*precent + m.deal_price
//// }else if(m.unit_price < m.deal_price){
//// newPrice = m.deal_price - (m.deal_price*precent)
//// }
//// if(newPrice){
//// updateDealPrice(m, newPrice.toInteger())
//// }
//// }
//// }
//
//// if(d['M']){
//// def rets = findThirdByPe(d['A'], yearMonth, d['M'])
//// for(def r : rets){
//// deletThirdPrice(r, thirdMonth)
//// }
//// }
// }
def list = findAllThirdByPe(yearMonth, 70)
for(def r : list){
deletThirdPrice(r, thirdMonth)
}
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\估值模型测试202201_1221 - 副本.xlsx",0,1)
for(def d : datas){
// if(d['G']){
// def rets = findDealByPe(d['A'], yearMonth, d['G'])
// for(def r : rets){
// deleteDealPrice(r)
// }
// }
// if(d['H']){
// def rets = findCityByPpeRange(d['A'], yearMonth, 15, 20)
// for(def m : rets){
// def precent = d['H']/100.0
// def newPrice = null
// if(m.unit_price > m.deal_price){
// newPrice = m.deal_price*precent + m.deal_price
// }else if(m.unit_price < m.deal_price){
// newPrice = m.deal_price - (m.deal_price*precent)
// }
// if(newPrice){
// updateDealPrice(m, newPrice.toInteger())
// }
// }
// }
// if(d['M']){
// def rets = findThirdByPe(d['A'], yearMonth, d['M'])
// for(def r : rets){
// deletThirdPrice(r, thirdMonth)
// }
// }
}
}
@@ -151,9 +152,10 @@ class DealPriceDataProcess {
}
def deleteDealPrice(r){
def sql = "delete from bigdata_deal_gj where commid='${r.commid}' and buildarea=${r.buildarea} and DEALUNITPRICE=${r.deal_price} " +
"and dealdate=to_date('${r.dealdate}','yyyy-mm-dd')"
sqlInstance236.executeUpdate(sql)
// def sql = "delete from bigdata_deal_gj where commid='${r.commid}' and buildarea=${r.buildarea} and DEALUNITPRICE=${r.deal_price} " +
// "and dealdate=to_date('${r.dealdate}','yyyy-mm-dd')"
def sql = "delete from base.t0831_bigdata_deal_gj_h where gujia_comm_id='${r.commid}' and txn_dt='${r.dealdate}' and guiabconstructionarea=${r.buildarea} and gujia_bds_trdtckt_prc=${r.deal_price}"
pgsqlInstance.executeUpdate(sql)
}
def deletThirdPrice(r, thirdMonth){
@@ -177,19 +179,14 @@ class DealPriceDataProcess {
}
def findAllDealByPe(yearMonth, peVal){
def sql = "select cityid,cityname,distid,commid,commname,unit_price,target_avg_price,pe,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate from avm_evaluation_item_gj where year_month=${yearMonth} and data_type='成交' and pe>${peVal}";
def sql = "select gujia_comm_id as commid,gujiabds_valt_unitprc as unit_price,gujiabds_to_mkprc_mny as target_avg_price,gujia_area as buildarea,to_char(mdl_dt,'yyyy-mm-dd') as dealdate from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearMonth} and gujia_data_tp='成交' and gujia_mns_rto>${peVal} ";
def rets = []
sqlInstance236.eachRow(sql,{r->
pgsqlInstance.eachRow(sql,{r->
def map = [:]
map.cityname = r.cityname
map.cityid=r.cityid
map.distid = r.distid
map.commid = r.commid
map.commname = r.commname
map.deal_price = r.target_avg_price
map.unit_price = r.unit_price
map.buildarea = r.buildarea
map.ppe = r.pe
map.dealdate = r.dealdate
rets << map
})
@@ -392,12 +389,12 @@ class DealPriceDataProcess {
def deleteThirdPartyData(){
def citys = CcbCommonUtils.citys_all
def month = 202309
def month = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//上个月日期
def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def set = new HashSet()
for(def city : citys) {
def rets = findPeGteThirdParty2(city, 202310)
def rets = findPeGteThirdParty2(city, DateUtils.formatDateByPattern(new Date(),"yyyyMM"))//当前月日期
def total = 0
println city +" "+rets.size()
if(rets){
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2023-10-27", "2023-11-25")
def prices = CcbCommonUtils.findPersonPrice(city, "2023-10-27", "2023-11-26")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){
def distMap = getDistrictMap(city)
@@ -1315,7 +1315,7 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20231124--批量修正价格.xlsx"
def fileName = "D:\\数据\\建行\\建行成交\\咸阳成交.xlsx"
ExcelUtils.multiThreadingModify(fileName, { d ->
// def count = CommonDatautils.findDistrictCount(null, d['B'], d['C'], "district")
// if(count && count==1){
@@ -1323,10 +1323,15 @@ ExcelUtils.multiThreadingModify(fileName, { d ->
// d['E'] = dist.city
// }
//
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
d['C'] = dist.region
if(d['F']){
def dist = CommonDatautils.findDistrictById(d['F'])
if(dist){
d['G'] = dist.status
d['H'] = dist.name_cat
d['I'] = dist.estate_type2
}
}
//// def newRegion = RegionData.formatRegion(d['A'], d['B'])
//// d['F'] = newRegion
},0)
@@ -3992,6 +3992,19 @@ class CommonDatautils {
return total;
}
static def findNewAmapDetailBySql(city){
String sql = "select * from amap_poi where cityname='${city}' and status=0"
def rets = []
ElasticUtils.executeSqlQuery(sql,{data->
def point = data.location
def location = [:]
location.lon = DataHandleUtils.getDoubleRound(6, point.x)
location.lat = DataHandleUtils.getDoubleRound(6, point.y)
rets << ["id":data.id,"province":data.pname,"city":data.cityname, "district":data.adname, "name":data.name, "address":data.address, "cat":data.type, "location":location]
})
return rets;
}
static def findAmapDetailBySql(city){
String sql = "select * from amap_poi_detail where cityname='${city}' and status=0"
def rets = []
@@ -39,10 +39,11 @@ class ApiTest {
//五级地址标准化
task.addrStructure("D:\\数据\\地数\\地址标准化-测试用例.xlsx")
// task.nineLevel("D:\\数据\\地数\\地址九级标准化-测试用例.xlsx")
}
def addrStructure(path){
ExcelUtils.modify(path,{d->
ExcelUtils.multiThreadingModify(path,{d->
if(d['B']){
JSONObject jsonObject = FangjiaClientUtils.formatFivelevel(d['A'], d['B']);
if(jsonObject) {
@@ -72,7 +73,86 @@ class ApiTest {
}
}
}
},4);
},7);
}
def nineLevel(path){
ExcelUtils.multiThreadingModify(path,{d->
if(d['B']){
def obj = FangjiaClientUtils.formatNinelevel(d['A'], d['B']);
if(obj) {
def jsonObject = obj.get("result")
if(jsonObject){
if(jsonObject.get("province") && jsonObject.get("province")!="null"){
d['C'] = jsonObject.get("province")
}
if(jsonObject.get("city") && jsonObject.get("city")!="null"){
d['D'] = jsonObject.get("city")
}
if(jsonObject.get("district") && jsonObject.get("district")!="null"){
d['E'] = jsonObject.get("district")
}
if(jsonObject.get("township") && jsonObject.get("township")!="null"){
d['F'] = jsonObject.get("township")
}
if(jsonObject.get("formatAddress") && jsonObject.get("formatAddress")!="null"){
d['G'] = jsonObject.get("formatAddress")
}
if(jsonObject.get("community") && jsonObject.get("community")!="null"){
d['H'] = jsonObject.get("community")
}
if(jsonObject.get("name") && jsonObject.get("name")!="null"){
d['I'] = jsonObject.get("name")
}
if(jsonObject.get("street") && jsonObject.get("street")!="null"){
d['J'] = jsonObject.get("street")
}
if(jsonObject.get("streetNumber") && jsonObject.get("streetNumber")!="null"){
d['K'] = jsonObject.get("streetNumber")
}
if(jsonObject.get("buildingNumber") && jsonObject.get("buildingNumber")!="null"){
d['L'] = jsonObject.get("buildingNumber")
}
if(jsonObject.get("unitNumber") && jsonObject.get("unitNumber")!="null"){
d['M'] = jsonObject.get("unitNumber")
}
if(jsonObject.get("roomNumber") && jsonObject.get("roomNumber")!="null"){
d['N'] = jsonObject.get("roomNumber")
}
if(jsonObject.get("floor") && jsonObject.get("floor")!="null"){
d['O'] = jsonObject.get("floor")
}
if(jsonObject.get("provinceCode") && jsonObject.get("provinceCode")!="null"){
d['P'] = jsonObject.get("provinceCode")
}
if(jsonObject.get("cityCode") && jsonObject.get("cityCode")!="null"){
d['Q'] = jsonObject.get("cityCode")
}
if(jsonObject.get("districtCode") && jsonObject.get("districtCode")!="null"){
d['R'] = jsonObject.get("districtCode")
}
def map = jsonObject.get("addressAssessment")
if(map){
d['S'] = map.get("msg")
}
}
}
}
},0);
}
def addressEvaluate(path){
@@ -327,4 +327,24 @@ class FangjiaClientUtils {
return jsonObj;
}
public static def formatNinelevel(city, address){
def jsonObj = null
def content = null
try {
address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaAddressService/structureAndComplete?address="+URLEncoder.encode(address,"UTF-8")+"&complete=true&addressAssessment=true"
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
println jsonObj
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
}
@@ -92,12 +92,13 @@ class GaoDePoiDataProcess extends BaseBatch{
}
def synchMapPoi(){
//"三明市","漳州市","厦门市","福州市"
//三明市,"宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市","重庆市"
def citys = ["上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市"]
//三明市,"宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市","重庆市" //厦门
//"上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市" //上海信用卡
def citys = ["杭州市","宁波市","温州市","嘉兴市","台州市","湖州市","绍兴市","金华市","舟山市","衢州市","丽水市"]
for(def city : citys){
def townMap = findTownRange(city)
def results = CommonDatautils.findAmapDetailBySql(city)
// def results = CommonDatautils.findAmapDetailBySql(city)
def results = CommonDatautils.findNewAmapDetailBySql(city)
println "city: "+city +" town: "+townMap.size()+" poi:"+results.size()
def rows = running(city, results, "map", townMap)
// save(city, rows)
@@ -221,7 +222,7 @@ class GaoDePoiDataProcess extends BaseBatch{
def fjw_province = province
def fjw_city = RegionData.formatCity(map.city)
def fjw_region = RegionData.formatRegion(fjw_city, map.district)
map.id = new ObjectId().toString()
// map.id = new ObjectId().toString()
map.fjw_province = fjw_province
map.fjw_city = fjw_city
@@ -161,6 +161,7 @@ class SeaweedDataSynch extends BaseBatch{
}else{
map.alias= dist.mapping_alias
}
}
if(dist.addr_alias){