Merge branch 'script_gj' into 'master'

update

See merge request !781
This commit is contained in:
2023-12-28 15:14:36 +08:00
9 changed files with 119 additions and 59 deletions
@@ -37,7 +37,7 @@ class DealPriceDataProcess {
}
def delDealByPe(){
def yearMonth = 202311
def yearMonth = 202312
def thirdMonth = 202112
def rets = findAllDealByPe(yearMonth, 18)
@@ -179,7 +179,7 @@ class DealPriceDataProcess {
}
def findAllDealByPe(yearMonth, peVal){
def sql = "select gujia_comm_id as commid,gujiabds_valt_unitprc as unit_price,gujiabds_to_mkprc_mny as target_avg_price,gujia_area as buildarea,to_char(mdl_dt,'yyyy-mm-dd') as dealdate from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearMonth} and gujia_data_tp='成交' and gujia_mns_rto>${peVal} ";
def sql = "select gujia_comm_id as commid,gujiabds_valt_unitprc as unit_price,gujiabds_to_mkprc_mny as target_avg_price,gujia_area as buildarea,to_char(mdl_dt,'yyyy-mm-dd') as dealdate from t_app_p9pha_plda.avm_evaluation_item_gj_h where urbn_nm in ('昆明','南京','苏州','重庆') and gujiabds_clc_yrmo_val=${yearMonth} and gujia_data_tp='成交' and gujia_mns_rto>${peVal} ";
def rets = []
pgsqlInstance.eachRow(sql,{r->
def map = [:]
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2023-10-27", "2023-11-26")
def prices = CcbCommonUtils.findPersonPrice(city, "2023-11-26", "2023-12-27")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){
def distMap = getDistrictMap(city)
@@ -1287,55 +1287,57 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
//}
//
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\指导价(保亭-文昌-詹州).xlsx"
//
//ExcelUtils.multiThreadingModify(fileName, {d->
//// if(d['A']){
////// def region = RegionData.formatRegion(d['B'], d['C'])
////// if(region){
////// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
////// if(dist){
////// d['G'] = dist.region
////// d['H'] = dist._id.toString()
////// }
////// }
// if(!d['G']){
// def dist = CommonDatautils.findDistrictById(d['A'])
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20231222--修正人工指导价.xlsx"
ExcelUtils.multiThreadingModify(fileName, {d->
if(d['A']){
// def region = RegionData.formatRegion(d['B'], d['C'])
// if(region){
// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
// if(dist){
// d['G'] = dist.region
// d['H'] = dist.name
// d['I'] = dist.status
// def val = d['E'].toString().toDouble().toInteger() + RandomUtil.random(-200,200)
// d['J'] = val.toInteger()
// d['H'] = dist._id.toString()
// }
// }
//// }
//},0)
def fileName = "D:\\数据\\建行\\建行成交\\咸阳成交.xlsx"
ExcelUtils.multiThreadingModify(fileName, { d ->
// def count = CommonDatautils.findDistrictCount(null, d['B'], d['C'], "district")
// if(count && count==1){
// def dist = CommonDatautils.findDistrict(null, d['B'], d['C'], "district")
// d['E'] = dist.city
// }
//
if(d['F']){
def dist = CommonDatautils.findDistrictById(d['F'])
if(dist){
d['G'] = dist.status
d['H'] = dist.name_cat
d['I'] = dist.estate_type2
if(!d['I']){
def dist = CommonDatautils.findDistrictById(d['C'])
if(dist){
d['I'] = dist.region
// d['H'] = dist.name
d['J'] = dist.status
if(d['A']=="天门" || d['A']=="迪庆"){
def val = d['E'].toString().toDouble().toInteger() + RandomUtil.random(-200,200)
d['K'] = val.toInteger()
}
}
}
}
//// def newRegion = RegionData.formatRegion(d['A'], d['B'])
//// d['F'] = newRegion
},0)
//def fileName = "D:\\数据\\建行\\建行成交\\咸阳成交.xlsx"
//ExcelUtils.multiThreadingModify(fileName, { d ->
//// def count = CommonDatautils.findDistrictCount(null, d['B'], d['C'], "district")
//// if(count && count==1){
//// def dist = CommonDatautils.findDistrict(null, d['B'], d['C'], "district")
//// d['E'] = dist.city
//// }
////
// if(d['F']){
// def dist = CommonDatautils.findDistrictById(d['F'])
// if(dist){
// d['G'] = dist.status
// d['H'] = dist.name_cat
// d['I'] = dist.estate_type2
// }
// }
//
////// def newRegion = RegionData.formatRegion(d['A'], d['B'])
////// d['F'] = newRegion
//},0)
//ExcelUtils.modify(fileName, {d->
// if(d['A']){
// def sql = "update guided_price_gj set dealdate=to_date('2022-09-20','yyyy-MM-dd') where id='${d['A']}'"
@@ -93,13 +93,13 @@ class TradeStatLog {
def run(){
def citys = CcbCommonUtils.citys_all;
def rets = []
rets << ["城市","23年7月总量","23年8月总量","23年9月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
rets << ["城市","23年9月总量","23年10月总量","23年11月总量","占比","贝壳","链家", "我爱我家","幸福里APP","中原"]
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2023-05-01"
def date = "2023-08-01"
def endDate = "2023-09-01"
def newDate = "2023-10-01"
def startDate = "2023-09-01"
def date = "2023-10-01"
def endDate = "2023-11-01"
def newDate = "2023-12-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -4001,7 +4001,7 @@ class CommonDatautils {
def location = [:]
location.lon = DataHandleUtils.getDoubleRound(6, point.x)
location.lat = DataHandleUtils.getDoubleRound(6, point.y)
rets << ["id":data.id,"province":data.pname,"city":data.cityname, "district":data.adname, "name":data.name, "address":data.address, "cat":data.type, "location":location]
rets << ["data_id":data.id,"province":data.pname,"city":data.cityname, "district":data.adname, "name":data.name, "address":data.address, "cat":data.type, "location":location]
})
return rets;
}
@@ -605,6 +605,27 @@ class OpenApiUtils {
return null;
}
static def houseValueInformation(city, address, area){
address = address.replace(" ","").replace("%","")
def url = "http://open.fangjia.com/comprehensiveQuery/houseValueInformation?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
if(area){
url = url +"&size="+area
}
// println url
String content = new HttpRequest().get(url, "utf-8");
def dto = []
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==200 && doc.result){
def dtos = doc.result
return dtos
}else{
println url
}
}
return null;
}
static def financialRating(city, address){
def url = "http://open.fangjia.com/address/financialRating?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
String content = new HttpRequest().get(url, "utf-8");
@@ -48,11 +48,11 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2023-10-01"
static def statDate = "2023-11-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
// task.esfParser("D:\\数据\\友达\\2023年11月\\查询1.DBF");//友达二手房存入mongo
// task.esfParser("D:\\数据\\友达\\2023年12月\\查询1.DBF");//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
@@ -36,9 +36,10 @@ class ApiTest {
// task.addressEvaluate("D:\\数据\\北交所\\拍卖案例(-地址均价接口).xlsx")//地址估值
// task.arroundFacilitiesByAddress("D:\\数据\\北交所\\拍卖案例(-周边poi接口).xlsx")//北交所周边poi 地址
// task.jsAbcHouseCastTestByAddress("D:\\数据\\北交所\\拍卖案例(房源接口500条-价格控20%).xlsx")
task.houseValueInformation("D:\\数据\\北交所\\北交所综合接口测试.xlsx")
//五级地址标准化
task.addrStructure("D:\\数据\\地数\\地址标准化-测试用例.xlsx")
// task.addrStructure("D:\\数据\\地数\\地址标准化-测试用例.xlsx")
// task.nineLevel("D:\\数据\\地数\\地址九级标准化-测试用例.xlsx")
}
@@ -516,4 +517,40 @@ class ApiTest {
}
def houseValueInformation(path){
ExcelUtils.multiThreadingModify(path,{d->
def results = com.fangjia.script.gj.OpenApiUtils.houseValueInformation(d['A'], d['B'], d['C'])
if(results) {
def addressValue = results.addressValue
if (addressValue) {
d['D'] = addressValue.city + "-" + addressValue.district + "-" + addressValue.name + "-" + addressValue.avgPrice + "-" + addressValue.totalPrice
}
def communityAvgPrice = results.communityAvgPrice
if (communityAvgPrice) {
d['E'] = communityAvgPrice.toString()
}
def surroundingPoi = results.surroundingPoi
if (surroundingPoi) {
d['F'] = surroundingPoi.toString()
}
def house = results.houseCase
if (house) {
d['G'] = house.minUnitPrice
d['H'] = house.maxUnitPrice
def datas = house.datas
if (datas) {
def str = ""
for (def r : datas) {
str = str + r.buildingName + "-" + r.strctureArea + "-" + r.unitPrice + "-" + r.sampleDate.substring(0, 10) + "\n"
}
d['J'] = str
}
}
}
})
}
}
@@ -50,7 +50,7 @@ class GaoDePoiDataProcess extends BaseBatch{
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL)
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_new", MAP_POI_URL)
DBCollection MAPPOI_CONN_TEST = MongoUtils.getCollectionByName2("fangjia", "map_poi_all_8", MAP_POI_URL)
@@ -66,8 +66,8 @@ class GaoDePoiDataProcess extends BaseBatch{
@Override
void process(Object args) {
// println ProvinceData.getProvince("成都")
// synchMapPoi();
synchCompany()
synchMapPoi();
// synchCompany()
// poiCatProcess()
// addressAndCatProcess()
// synchNetwork()
@@ -125,10 +125,10 @@ class GaoDePoiDataProcess extends BaseBatch{
}
def synchMapPoi(){
def runedCity = ["三明市","宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市","重庆市","上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市" , "杭州市","宁波市","温州市","嘉兴市","台州市","湖州市","绍兴市","金华市","舟山市","衢州市","丽水市"]
def runedCity = []
fullCityMap = CommonDatautils.queryFullNameCity()
def citys = CommonDatautils.findAllCity();
def citys = CommonDatautils.findAllCity().reverse();
for(def sampleCity : citys){
def cityObj = fullCityMap.get(sampleCity);
def city = null
@@ -269,7 +269,7 @@ class GaoDePoiDataProcess extends BaseBatch{
def fjw_province = province
def fjw_city = RegionData.formatCity(map.city)
def fjw_region = RegionData.formatRegion(fjw_city, map.district)
// map.id = new ObjectId().toString()
map.id = new ObjectId().toString()
map.fjw_province = fjw_province
map.fjw_city = fjw_city