This commit is contained in:
2023-11-20 17:02:16 +08:00
parent 226afe9bc4
commit c9355a86a1
7 changed files with 65 additions and 24 deletions
@@ -60,25 +60,16 @@ class BigdataDealBatch extends BaseBatch{
@Override
public void process(Object args) {
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
// def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.getRunningCitys(args)
// //'广州','重庆','南京','石家庄','廊坊','佛山','无锡',
// citys = ['哈尔滨']
// if(args && args.size()>=2){
// curDate = args[1].toString().toInteger()
// }
// println "curDate:"+curDate
// def allCitys = CommonDatautils.findAllCity()
// allCitys = ['哈尔滨']
// for(def city : allCitys){
//// if(!ccbCitys.contains(city)){
// addDealPriceMatch(city);
//// }
// }
//
//
println "curDate:"+curDate
def allCitys = CommonDatautils.findAllCity()
for(def city : allCitys){
addDealPriceMatch(city);
}
def rows = []
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
@@ -119,6 +110,7 @@ class BigdataDealBatch extends BaseBatch{
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成 " + rows.size()
delPgData()
saveToPgSql(rows)
// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat")
@@ -742,6 +734,14 @@ class BigdataDealBatch extends BaseBatch{
println city +" "+trades.size()+" "+new Date().toString()
}
def delPgData(){
def month = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
pgsqlInstance.execute("delete from base.t0831_bigdata_deal_gj_h where p9_start_date='${month}'")
println "base.t0831_bigdata_deal_gj_h 数据已删除"
}
def saveToPgSql(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
@@ -27,8 +27,8 @@ class BigdataSynchPrice extends BaseBatch{
def citys = CcbCommonUtils.getRunningCitys(args)
// synchCityPrice(citys)
// synchDistrictPrice(citys)
synchCityPrice(citys)
synchDistrictPrice(citys)
synchCommPirce(citys)
}
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2023-10-27", "2023-11-07")
def prices = CcbCommonUtils.findPersonPrice(city, "2023-10-27", "2023-11-18")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){
def distMap = getDistrictMap(city)
@@ -85,7 +85,7 @@ class GuiDingPriceBatch extends BaseBatch{
saveToPgSql(rows)
//
//更新静态数据表
// updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
}
@@ -48,11 +48,11 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2023-09-01"
static def statDate = "2023-10-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
// task.esfParser("D:\\数据\\友达\\2023年10月\\查询1.DBF");//友达二手房存入mongo
// task.esfParser("D:\\数据\\友达\\2023年11月\\查询1.DBF");//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
@@ -34,7 +34,28 @@ class ApiTest {
// task.addressAvgPrice("D:\\数据\\北交所\\拍卖案例(-地址估值接口).xlsx")//地址出小区均价
// task.addressEvaluate("D:\\数据\\北交所\\拍卖案例(-地址均价接口).xlsx")//地址估值
// task.arroundFacilitiesByAddress("D:\\数据\\北交所\\拍卖案例(-周边poi接口).xlsx")//北交所周边poi 地址
task.jsAbcHouseCastTestByAddress("D:\\数据\\北交所\\拍卖案例(房源接口500条-价格控20%).xlsx")
// task.jsAbcHouseCastTestByAddress("D:\\数据\\北交所\\拍卖案例(房源接口500条-价格控20%).xlsx")
//五级地址标准化
task.addrStructure("D:\\数据\\地数\\地址标准化-测试用例.xlsx")
}
def addrStructure(path){
ExcelUtils.modify(path,{d->
if(d['B']){
def jsonObject = FangjiaClientUtils.formatFivelevel(d['A'], d['B']);
if(jsonObject && jsonObject.containsKey("result")) {
d['C'] = jsonObject.get("province")
d['D'] = jsonObject.get("city")
d['E'] = jsonObject.get("district")
d['F'] = jsonObject.get("township")
d['G'] = jsonObject.get("address")
d['H'] = jsonObject.get("provinceCode")
d['I'] = jsonObject.get("cityCode")
d['J'] = jsonObject.get("districtCode")
}
}
},0);
}
def addressEvaluate(path){
@@ -308,4 +308,22 @@ class FangjiaClientUtils {
return jsonObj;
}
public static def formatFivelevel(city, address){
def jsonObj = null
def content = null
try {
address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
}
@@ -1548,4 +1548,6 @@ public class OpenApiClient {
}
}