diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/FindStageDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/FindStageDistrictBatch.groovy index c2f4eaee4..8c6633b4d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/FindStageDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/FindStageDistrictBatch.groovy @@ -9,7 +9,7 @@ import com.mongodb.BasicDBObject import org.apache.commons.lang.StringUtils import java.util.regex.Pattern - +// 分期分区与主小区映射关系表 @BatchOwner(Watcher.WangWenYan) class FindStageDistrictBatch extends BaseBatch { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 3e18b6b7f..83c9ec794 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -678,11 +678,7 @@ class CcbCommonUtils { } public static def deleteGuidedPrice(def dealdate){ - def sql = "delete from guided_price_gj where dealdate=to_date('${dealdate}','yyyy-mm-dd')" - sqlInstance.executeUpdate(sql) - - - def sql2 = "delete from base.t0831_guided_price_gj_h where gujia_drc_dt='${dealdate}'" + def sql2 = "delete from base.guided_price where gujia_drc_dt='${dealdate}'" pgsqlInstance.executeUpdate(sql2) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 5234f7fd4..c42ab1349 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -1289,7 +1289,7 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ -def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240508--人工指导价.xls" +def fileName = "D:\\数据\\建行\\房产估值\\指导价\\5月国信达价格录入.xlsx" ExcelUtils.multiThreadingModify(fileName, {d-> if(d['J']){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index fde3609ec..ea417b329 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1890,6 +1890,25 @@ class CommonDatautils { def dbo = null; while(cur.hasNext()) { dbo = cur.next() + if(dbo.value_type == 1){ + dbo.cal_type = "指导价" + }else if(dbo.value_type == 2){ + dbo.cal_type = "自身样本" + }else if(dbo.value_type == 3){ + dbo.cal_type = "第三方" + }else if(dbo.value_type == 4){ + dbo.cal_type = "交易数据" + }else if(dbo.value_type == 5){ + dbo.cal_type = "周边楼盘" + }else if(dbo.value_type == 6){ + dbo.cal_type = "推导" + }else if(dbo.value_type == 9){ + dbo.cal_type = "神经网络" + }else if(dbo.value_type == 10){ + dbo.cal_type = "无方式" + }else{ + dbo.cal_type = "" + } } return dbo; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy index 2b96edc91..d4e07eab4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy @@ -22,8 +22,8 @@ class FzDistrictMatch { // task.addPrice() // task.addThirdPrice() -// task.addThirdPrice2("D:\\数据\\福州\\价格比对_20240315.xlsx") - task.longyanPrice() + task.addThirdPrice2("D:\\数据\\福州\\价格比对_20240315.xlsx") +// task.longyanPrice() } def longyanPrice(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index f24f72702..420fa5c2d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -29,7 +29,7 @@ class DevProcess { def task = new DevProcess(); // task.run() // task.parseCityAndDist() -// task.devSplitRoad() + task.devSplitRoad() // task.addFlag() // task.matchDev() @@ -42,7 +42,8 @@ class DevProcess { // task.addNewDev() // task.createDevMapping() // task.deleteAddr() - task.synchDevTo43() +// task.synchDevTo43() + } def parseCityAndDist(){ @@ -159,54 +160,69 @@ class DevProcess { } def devSplitRoad(){ - ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\企业园区地址-新.xlsx",{d-> - /*if(d['C'] && !d['V']){ - def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['E'], d['C']) - def street = jsonObject.get("street") - def streetNumber = jsonObject.get("streetNumber") - if(jsonObject.get("province") && jsonObject.get("province")!="null"){ - d['V'] = jsonObject.get("province") - } - if(jsonObject.get("city") && jsonObject.get("city")!="null"){ - d['W'] = jsonObject.get("city") - } + def fileName = "/home/goojia/第二批公司.xlsx" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\数据建设\\第二批公司.xlsx" + } - if(jsonObject.get("district") && jsonObject.get("district")!="null"){ - d['X'] = jsonObject.get("district") - } + def i = 5; + while(i<13){ + ExcelUtils.multiThreadingModify(fileName,{d-> + /*if(d['C'] && !d['V']){ + def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['E'], d['C']) + def street = jsonObject.get("street") + def streetNumber = jsonObject.get("streetNumber") + if(jsonObject.get("province") && jsonObject.get("province")!="null"){ + d['V'] = jsonObject.get("province") + } - if(jsonObject && street && streetNumber && street!="null"){ - d['Y'] = jsonObject.get("street") - if(streetNumber!='null'){ - d['Z'] = jsonObject.get("streetNumber") + if(jsonObject.get("city") && jsonObject.get("city")!="null"){ + d['W'] = jsonObject.get("city") + } + + if(jsonObject.get("district") && jsonObject.get("district")!="null"){ + d['X'] = jsonObject.get("district") + } + + if(jsonObject && street && streetNumber && street!="null"){ + d['Y'] = jsonObject.get("street") + if(streetNumber!='null'){ + d['Z'] = jsonObject.get("streetNumber") + } + } + }*/ + if(d['C'] && d['B']!='河北省'){ + def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(null, d['C']) + def street = jsonObject.get("street") + def streetNumber = jsonObject.get("streetNumber") + if(jsonObject.get("province") && jsonObject.get("province")!="null"){ + d['J'] = jsonObject.get("province") + } + + if(jsonObject.get("city") && jsonObject.get("city")!="null"){ + d['K'] = jsonObject.get("city") + } + + if(jsonObject.get("district") && jsonObject.get("district")!="null"){ + d['L'] = jsonObject.get("district") + } + + if(jsonObject && street && streetNumber && street!="null"){ + d['M'] = jsonObject.get("street") + if(streetNumber!='null'){ + d['N'] = jsonObject.get("streetNumber") + } + } + + if(jsonObject.get("dev") && jsonObject.get("dev")!="null"){ + d['O'] = jsonObject.get("dev") } } - }*/ - if(d['D'] && !d['M'] && d['P'] && d['P'].toString().toDouble().toInteger()==1){ - def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['E'], d['D']) - def street = jsonObject.get("street") - def streetNumber = jsonObject.get("streetNumber") - if(jsonObject.get("province") && jsonObject.get("province")!="null"){ - d['J'] = jsonObject.get("province") - } + },i) + i++ + } - if(jsonObject.get("city") && jsonObject.get("city")!="null"){ - d['K'] = jsonObject.get("city") - } - - if(jsonObject.get("district") && jsonObject.get("district")!="null"){ - d['L'] = jsonObject.get("district") - } - - if(jsonObject && street && streetNumber && street!="null"){ - d['M'] = jsonObject.get("street") - if(streetNumber!='null'){ - d['N'] = jsonObject.get("streetNumber") - } - } - } - },0) } def addFlag(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictAliasProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictAliasProcess.groovy index 289e6f68f..c987e3b7c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictAliasProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictAliasProcess.groovy @@ -14,7 +14,7 @@ class DistrictAliasProcess { } def run(){ - ExcelUtils.modify("D:\\数据\\建行\\房产估值\\名称在映射别名中的小区(2).xlsx",{ d-> + ExcelUtils.modify("D:\\数据\\数据建设\\小区\\名称在映射别名中-第二批.xlsx",{ d-> if(d['H']=="是"){ def udata = new BasicDBObject() udata.put("status", 1) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy index ea2bd7f20..32e76368d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy @@ -10,6 +10,7 @@ import com.mongodb.DBCollection import com.mongodb.DBObject import com.mongodb.client.AggregateIterable import org.bson.Document +import org.bson.types.ObjectId /** * 乡镇街道中间表 @@ -35,6 +36,7 @@ class TownMiddleProcess { static main(args) { def task = new TownMiddleProcess(); task.run() + } def run(){ //创建省份 @@ -52,6 +54,13 @@ class TownMiddleProcess { //添加地级市标志 // addCityFlag() // addPinyin() + //打标记,将开发区与民政部相同的打上1 +// addTjjDevFlag() + //打标记,用于标识乡镇街道是否输出及格式化输出街道 +// addStreetFlag() + +// addDistrictDiffFlag() + } @@ -751,4 +760,44 @@ class TownMiddleProcess { } } + + def addTjjDevFlag(){ + def q = new BasicDBObject("type", "区域") + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(2000) + for(def d : dbcursor){ + if(d.district && d.mzb_district_name && d.district==d.mzb_district_name){ + def udata = new BasicDBObject() + udata.put("district_diff", 1) //民政部,统计局 名称相同 district_diff打1 + ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata)) + } + } + } + + def addStreetFlag(){ + ExcelUtils.modify("D:\\数据\\数据建设\\乡镇街道\\乡镇街道-结果.xlsx",{ row-> + if(row['I']){ + if(row['I']=="标0"){ + def udata = new BasicDBObject() + udata.put("street_output", 0) //0:不输出 1:输出 + ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",new ObjectId(row['A'])), new BasicDBObject('$set',udata)) + }else if(row['I']=="标1"){ + def udata = new BasicDBObject() + udata.put("street_output", 1) //0:不输出 1:输出 + if(row['G']){ + udata.put("street", row['G']) //街道中去除区域 + } + ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",new ObjectId(row['A'])), new BasicDBObject('$set',udata)) + } + } + }); + } + + def addDistrictDiffFlag(){ + ExcelUtils.modify("D:\\数据\\数据建设\\区域-结果.xlsx",{ row-> + def flag = row['F'].toString().toDouble().toInteger() + def udata = new BasicDBObject() + udata.put("district_diff", flag) + ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",new ObjectId(row['A'])), new BasicDBObject('$set',udata)) + }); + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 0afe56328..71132614c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -354,13 +354,13 @@ class FangjiaClientUtils { def content = null try { address = address.replace("{","") - def url = "http://192.168.0.19:31080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=true&addressAssessment=true" + def url = "http://192.168.0.19:31080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true" if(city){ url = url +"&city="+URLEncoder.encode(city,"UTF-8") } content = new HttpRequest().newGet(url, null, "utf-8") jsonObj = JSONObject.fromObject(content) - println jsonObj +// println jsonObj } catch (Exception e) { println address +" "+content e.printStackTrace(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataDealBatch.groovy index ebb2ccf1f..21bb3652d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataDealBatch.groovy @@ -48,11 +48,9 @@ class FjwBigdataDealBatch extends BaseBatch{ static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", - "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") - def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified", - "postgres", "postgres", "org.postgresql.Driver") + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified", + "postgres", "Goojia123456", "org.postgresql.Driver") static main(args) { runBatch(new FjwBigdataDealBatch(), args); @@ -61,59 +59,50 @@ class FjwBigdataDealBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.getRunningCitys(args) + def citys = ["乌鲁木齐"] //CcbCommonUtils.getRunningCitys(args) + +// println "curDate:"+curDate +// def allCitys = CommonDatautils.findAllCity() +// for(def city : allCitys){ +// addDealPriceMatch(city); +// } + + def rows = [] + def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def index = 0 + for(def city : citys){ + index ++ + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + def ppeDists = [:] + + def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city)) + println "历史id加载量:"+dealIds.size() + def statMap = [:]//getDistrictStatByCity(city, weekend) + if(city=="上海"){ + def youDataTrade = getYouDaData(statMap) + def youdaDatas = createData(city, youDataTrade, cityCodeDataMap, distCodeMap, "youda", dealIds) + rows.addAll(youdaDatas) + println city +" "+youdaDatas.size() + } + + def trades = getCityData(city, statMap, ppeDists) + if(trades){ + def rets = createData(city, trades, cityCodeDataMap, distCodeMap, "other", dealIds) + rows.addAll(rets) + println "【${index}】"+city +" 成交共:"+rets.size() + + def cityDatas = [] + cityDatas << headers + cityDatas.addAll(rows) + } - println "curDate:"+curDate - def allCitys = CommonDatautils.findAllCity() - for(def city : allCitys){ - addDealPriceMatch(city); } -// def rows = [] -// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] -// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() -// def index = 0 -// for(def city : citys){ -// index ++ -// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) -// def ppeDists = [:] -// -// def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city)) -// println "历史id加载量:"+dealIds.size() -// def statMap = [:]//getDistrictStatByCity(city, weekend) -// if(city=="上海"){ -// def youDataTrade = getYouDaData(statMap) -// def youdaDatas = createData(city, youDataTrade, cityCodeDataMap, distCodeMap, "youda", dealIds) -// rows.addAll(youdaDatas) -// println city +" "+youdaDatas.size() -// } -// -// def trades = getCityData(city, statMap, ppeDists) -// if(trades){ -// def rets = createData(city, trades, cityCodeDataMap, distCodeMap, "other", dealIds) -// rows.addAll(rets) -// println "【${index}】"+city +" 成交共:"+rets.size() -// -// def cityDatas = [] -// cityDatas << headers -// cityDatas.addAll(rows) -// } -// -// } -// -// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" -// if(CommonDatautils.isWindow()){ -// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" -// } -// -// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); -// println "txt 生成 " + rows.size() -// -//// delPgData() -// saveToPgSql(rows) -// -//// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat") -//// saveToPgSql2(datas) + saveToPgSql(rows) + +// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat") +// saveToPgSql2(datas) } def readTxt(def fileName){ @@ -186,19 +175,7 @@ class FjwBigdataDealBatch extends BaseBatch{ } - def getCommunityPrices(def city){ - def sql = "select commid, avg_price from community_base_price_gj where year_month=${endDate} and cityname='${city}'"; - def map = [:] - try{ - sqlInstance.eachRow(sql,{ - map.put(it.commid, it.avg_price) - }); - }catch(Exception e){ - getPpeGTE15Datas(city) - } - return map - } def findThirdPartyPrice(def city) { def q = new BasicDBObject() @@ -529,7 +506,7 @@ class FjwBigdataDealBatch extends BaseBatch{ def getCityData(def city, def statMap, def ppeDistsMap){ - def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"], null) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"], "all") println city +" 小区量:"+dists.size() def tradesList = Collections.synchronizedList(new ArrayList()); def totalCount = new AtomicLong() @@ -537,9 +514,13 @@ class FjwBigdataDealBatch extends BaseBatch{ // def citys = ["北京","广州","深圳","重庆","郑州","西安","成都","东莞","佛山","杭州","合肥","南京","长春","扬州","无锡","太原","台州","泉州","宁波","南通","南昌","贵阳","哈尔滨","惠州","济南","金华","昆明","遵义","镇江","肇庆","宜昌","芜湖","泰州","汕头","清远","秦皇岛","马鞍山","洛阳","连云港","九江","淮安","呼和浩特","沧州","保定","安庆"] - def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) +// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) +// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date()) + + def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20200101", "yyyyMMdd")) + def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20240501", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date()) + - def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date()) for(def i=0;i for(def r : rows) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwGuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwGuiDingPriceBatch.groovy new file mode 100644 index 000000000..8e494f407 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwGuiDingPriceBatch.groovy @@ -0,0 +1,226 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.gj.fjwPriceBaseData.FjwPgsqlCommUtils +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import groovy.sql.Sql + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import org.bson.types.ObjectId +/** + * 建行模型估值项目 + * 人工指导价同步 + */ + +@BatchOwner(Watcher.GaoJuan) +class FjwGuiDingPriceBatch extends BaseBatch{ + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); + def static final DBCollection dataProcessColl = CrawalDBUtil.getDataProcessCol(); + + def curDate= DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-15"; + + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified", + "postgres", "Goojia123456", "org.postgresql.Driver") + + + static main(args) { + runBatch(new FjwGuiDingPriceBatch(), args); + } + + @Override + public void process(Object args) { + def citys = CcbCommonUtils.getRunningCitys(args) + def rows = [] + + FjwPgsqlCommUtils.deleteGuidedPrice(curDate) + for (def city : citys) { + def cityCodeDatas = CcbCommonUtils.findCityCodeValue() + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + + //添加自己库中的人工指导价 + def prices = CcbCommonUtils.findPersonPrice(city, "2022-03-01", "2022-04-01") + if(prices){ + def distMap = getDistrictMap(city) + def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) + rows.addAll(rets) + println city + " " + rets.size() + } + } + saveToPgSql(rows) + } + + def findFangDatas(def districts){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + println " 小区量:"+districts.size() + def result = Collections.synchronizedList(new ArrayList()); + for(def i=0;i + for(def r : rows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20]) + } + } + + println "end: "+new Date() + } + + + + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwPgsqlCommUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwPgsqlCommUtils.groovy index 10c94f0ea..21ecf0551 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwPgsqlCommUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwPgsqlCommUtils.groovy @@ -21,5 +21,19 @@ class FjwPgsqlCommUtils { return set } + public static def findDistGovPrice(def city){ + def ids = [] + sqlInstance.eachRow("SELECT gujia_bds_id from base.community_guide_price where urbn_nm='${city}'",{it-> + ids << it.gujia_bds_id + }); + return ids; + } + + + public static def deleteGuidedPrice(def dealdate){ + def sql2 = "delete from base.guided_price where gujia_drc_dt='${dealdate}'" + sqlInstance.executeUpdate(sql2) + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy new file mode 100644 index 000000000..76a828232 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy @@ -0,0 +1,674 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.gj.fjwPriceBaseData.FjwPgsqlCommUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import groovy.sql.Sql +import org.bson.types.ObjectId + +import java.text.SimpleDateFormat +import java.util.concurrent.ConcurrentHashMap + +/** + * 建行模型估值项目 + * 第三方均价数据同步 + */ +@BatchOwner(Watcher.GaoJuan) +class FjwThirdPartyPriceBatch extends BaseBatch { + + def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//202304// + def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302 + def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304 +// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); + + static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") + + def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07","淘房365":"08","中原":"09","兔博士":"10","Q房":"11","房探":"12", + "房多多":"13","我爱我家":"14","禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20"] + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified", + "postgres", "Goojia123456", "org.postgresql.Driver") + + static main(args) { + runBatch(new FjwThirdPartyPriceBatch(), args); + } + + @Override + public void process(Object args) { + + def citys = ["乌鲁木齐"]//CcbCommonUtils.getRunningCitys(args) + def stop = false + def month = -1 + while(!stop){ + + def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd")) + println weekend.toString() +" "+sDate + runDatas(citys) + month = month -1 + sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), month, "yyyyMM")); + if(sDate == 201912){ + stop = true; + } + } + } + + def runDatas(citys){ + def rows = [] + def index = 0 + def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd")) + def cityCodeDatas = CcbCommonUtils.findCityCodeValue() + for(def city : citys){ + index++ + + def gocIdsPrices = [] + if(city == "深圳"){ + gocIdsPrices = FjwPgsqlCommUtils.findDistGovPrice(city) + } + + def statMap = getDistrictStatByCity(city, weekend) + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"], "all") + def thirdPriceMap = findThirdPartyPrice(city, statMap) + + def rets = createCityDatas1(cityCodeDatas, distCodeMap, dists, thirdPriceMap, gocIdsPrices) + rows.addAll(rets) + println index+" "+city+" "+rets.size() +" "+rows.size() + } + + saveToPgSql(rows) + } + + + def createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap){ + def rets = [] + for(def d : dists){ + + def third = thirdPriceMap.get(d.region+"##"+d.name) + if(third){ + def data = distCodeMap.get(d.region) + if(data){ + def ret = [] + ret << third.id.toString() + ret << d._id.toString() + ret << d.name + ret << cityCodeDatas.get(d.city) + ret << d.city + ret << data.district_code + ret << data.district + ret << sDate + + if(third.zhugezhaofang_esf_price){ + ret << third.zhugezhaofang_esf_price + }else{ + ret << "" + } + if(third.anjuke_esf_price){ + ret << third.anjuke_esf_price + }else{ + ret << "" + } + if(third.soufang_esf_price){ + ret << third.soufang_esf_price + }else{ + ret << "" + } + + if(third.lianjia_esf_price){ + ret << third.lianjia_esf_price + }else{ + ret << "" + } + + if(third.cityhouse_esf_price){ + ret << third.cityhouse_esf_price + }else{ + ret << "" + } + + if(third.maitian_esf_price){ + ret << third.maitian_esf_price + }else{ + ret << "" + } + + if(third.beike_esf_price){ + ret << third.beike_esf_price + }else{ + ret << "" + } + + if(third.get("365house_esf_price")){ + ret << third.get("365house_esf_price") + }else{ + ret << "" + } + + if(third.zhongyuan_esf_price){ + ret << third.zhongyuan_esf_price + }else{ + ret << "" + } + + if(third.tuboshi_esf_price){ + ret << third.tuboshi_esf_price + }else{ + ret << "" + } + if(third.qfang_esf_price){ + ret << third.qfang_esf_price + }else{ + ret << "" + } + + if(third.fangtan_esf_price){ + ret << third.fangtan_esf_price + }else{ + ret << "" + } + + if(third.fangduoduo_esf_price){ + ret << third.fangduoduo_esf_price + }else{ + ret << "" + } + if(third.get("5i5j_esf_price")){ + ret << third.get("5i5j_esf_price") + }else{ + ret << "" + } + ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + rets << ret + } + } + + } + return rets; + } + + + def createCityDatas1(cityCodeDatas, distCodeMap, dists, thirdPriceMap, gocPrices){ + def allRows = [] + for(def d : dists){ + if(gocPrices && gocPrices.contains(d._id.toString())){ + continue; + } + def third = thirdPriceMap.get(d.region+"##"+d.name) + if(third){ + def data = distCodeMap.get(d.region) + if(data){ + def dataMap = [:] + dataMap.COMMID = d._id.toString() + dataMap.COMMNAME = d.name + dataMap.CITYID = cityCodeDatas.get(d.city) + dataMap.CITYNAME = d.city + dataMap.DISTID = data.district_code + dataMap.DISTNAME = data.district + dataMap.YEAR_MONTH = sDate + def price_map = [:] + dataMap.price_map = price_map + if(third.zhugezhaofang_esf_price){ + price_map.zhuge_avg_price = third.zhugezhaofang_esf_price + } + if(third.anjuke_esf_price){ + price_map.anjuke_avg_price = third.anjuke_esf_price + } + if(third.soufang_esf_price){ + price_map.fang_avg_price = third.soufang_esf_price + } + if(third.lianjia_esf_price){ + price_map.lianjia_avg_price = third.lianjia_esf_price + } + if(third.cityhouse_esf_price){ + price_map.cityhouse_avg_price = third.cityhouse_esf_price + } + if(third.maitian_esf_price){ + price_map.maitian_avg_price = third.maitian_esf_price + } + if(third.beike_esf_price){ + price_map.ke_avg_price = third.beike_esf_price + } + if(third.get("365house_esf_price")){ + price_map.taofang_avg_price = third.get("365house_esf_price") + } + if(third.zhongyuan_esf_price){ + price_map.zhongyuan_avg_price = third.zhongyuan_esf_price + } + if(third.tuboshi_esf_price){ + price_map.tbs_avg_price = third.tuboshi_esf_price + } + if(third.qfang_esf_price){ + price_map.qfang_avg_price = third.qfang_esf_price + } + if(third.fangtan_esf_price){ + price_map.fangtan_avg_price = third.fangtan_esf_price + } + if(third.fangduoduo_esf_price){ + price_map.fangduoduo_avg_price = third.fangduoduo_esf_price + } + if(third.get("5i5j_esf_price")){ + price_map.wiwj_avg_price = third.get("5i5j_esf_price") + } + dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + allRows << dataMap + } + } + } + + def results = [] + for(def m : allRows){ + def priceMap = m.price_map + priceMap.each{k,v-> + def ret = [] + ret << new ObjectId().toString() + ret << m.CITYID + ret << m.DISTID + ret << m.COMMID + ret << m.COMMNAME + + def web = "" + if(k=="zhuge_avg_price"){ + web = "诸葛找房" + }else if(k=="anjuke_avg_price"){ + web = "安居客" + }else if(k=="fang_avg_price"){ + web = "房天下" + }else if(k=="lianjia_avg_price"){ + web = "链家" + }else if(k=="cityhouse_avg_price"){ + web = "禧泰数据" + }else if(k=="maitian_avg_price"){ + web = "麦田" + }else if(k=="ke_avg_price"){ + web = "贝壳" + }else if(k=="taofang_avg_price"){ + web = "淘房365" + }else if(k=="zhongyuan_avg_price"){ + web = "中原" + }else if(k=="tbs_avg_price"){ + web = "兔博士" + }else if(k=="qfang_avg_price"){ + web = "Q房" + }else if(k=="fangtan_avg_price"){ + web = "房探" + }else if(k=="fangduoduo_avg_price"){ + web = "房多多" + }else if(k=="wiwj_avg_price"){ + web = "我爱我家" + } + + ret << sourceMap.get(web) + ret << web + ret << m.YEAR_MONTH + ret << v + ret << "" + ret << "" + ret << m.updatetime + results << ret + } + } + return results + } + + def createCityDatas2(cityCodeDatas, distCodeMap, dists, thirdPriceMap) { + def rets = [] + for (def d : dists) { + + def third = thirdPriceMap.get(d.region + "##" + d.name) + if (third) { + def data = distCodeMap.get(d.region) + if (data) { + third.each{k,v-> + def ret = [] + ret << new ObjectId().toString() + ret << cityCodeDatas.get(d.city) + ret << data.district_code + ret << d._id.toString() + ret << d.name + def web = "" + if(k=="zhugezhaofang_esf_price"){ + web = "诸葛找房" + }else if(k=="anjuke_esf_price"){ + web = "安居客" + }else if(k=="soufang_esf_price"){ + web = "房天下" + }else if(k=="lianjia_esf_price"){ + web = "链家" + }else if(k=="cityhouse_esf_price"){ + web = "禧泰数据" + }else if(k=="maitian_esf_price"){ + web = "麦田" + }else if(k=="beike_esf_price"){ + web = "贝壳" + }else if(k=="365house_esf_price"){ + web = "淘房365" + }else if(k=="zhongyuan_esf_price"){ + web = "中原" + }else if(k=="tuboshi_esf_price"){ + web = "兔博士" + }else if(k=="qfang_esf_price"){ + ret << "Q房" + }else if(k=="fangtan_esf_price"){ + ret << "房探" + }else if(k=="fangduoduo_esf_price"){ + ret << "房多多" + }else if(k=="5i5j_esf_price"){ + ret << "我爱我家" + }else{ + ret << "" + } + + ret << sourceMap.get(web) + ret << web + ret << sDate + ret << v + + ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + rets << ret + } + } + } + } + return rets; + } + + def createData3(cityid){ + def sql = "select COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, ZHUGE_AVG_PRICE, ANJUKE_AVG_PRICE, FANG_AVG_PRICE, LIANJIA_AVG_PRICE, CITYHOUSE_AVG_PRICE, MAITIAN_AVG_PRICE, KE_AVG_PRICE, TAOFANG_AVG_PRICE, ZHONGYUAN_AVG_PRICE, TBS_AVG_PRICE, QFANG_AVG_PRICE, FANGTAN_AVG_PRICE, FANGDUODUO_AVG_PRICE, WIWJ_AVG_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from THIRD_PARTY_PRICE_GJ where CITYID='${cityid}' and YEAR_MONTH=${sDate}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def dataMap = [:] + + dataMap.COMMID = r.COMMID + dataMap.COMMNAME = r.COMMNAME + dataMap.CITYID = r.CITYID + dataMap.CITYNAME = r.CITYNAME + dataMap.DISTID = r.DISTID + dataMap.DISTNAME = r.DISTNAME + dataMap.YEAR_MONTH = r.YEAR_MONTH + def price_map = [:] + dataMap.price_map = price_map + if(r.ZHUGE_AVG_PRICE){ + price_map.zhuge_avg_price = r.ZHUGE_AVG_PRICE + } + + if(r.ANJUKE_AVG_PRICE){ + price_map.anjuke_avg_price = r.ANJUKE_AVG_PRICE + } + + if(r.FANG_AVG_PRICE){ + price_map.fang_avg_price = r.FANG_AVG_PRICE + } + + if(r.LIANJIA_AVG_PRICE){ + price_map.lianjia_avg_price = r.LIANJIA_AVG_PRICE + } + + if(r.CITYHOUSE_AVG_PRICE){ + price_map.cityhouse_avg_price = r.CITYHOUSE_AVG_PRICE + } + + if(r.MAITIAN_AVG_PRICE){ + price_map.maitian_avg_price = r.MAITIAN_AVG_PRICE + } + + if(r.KE_AVG_PRICE){ + price_map.ke_avg_price = r.KE_AVG_PRICE + } + + if(r.TAOFANG_AVG_PRICE){ + price_map.taofang_avg_price = r.TAOFANG_AVG_PRICE + } + + if(r.ZHONGYUAN_AVG_PRICE){ + price_map.zhongyuan_avg_price = r.ZHONGYUAN_AVG_PRICE + } + + if(r.TBS_AVG_PRICE){ + price_map.tbs_avg_price = r.TBS_AVG_PRICE + } + + if(r.QFANG_AVG_PRICE){ + price_map.qfang_avg_price = r.QFANG_AVG_PRICE + } + + if(r.FANGTAN_AVG_PRICE){ + price_map.fangtan_avg_price = r.FANGTAN_AVG_PRICE + } + + if(r.FANGDUODUO_AVG_PRICE){ + price_map.fangduoduo_avg_price = r.FANGDUODUO_AVG_PRICE + } + + if(r.WIWJ_AVG_PRICE){ + price_map.wiwj_avg_price = r.WIWJ_AVG_PRICE + } + dataMap.updatetime = r.UPDATETIME + + allRows << dataMap + }); + + + def results = [] + for(def m : allRows){ + def priceMap = m.price_map + priceMap.each{k,v-> + def ret = [] + ret << new ObjectId().toString() + ret << m.CITYID + ret << m.DISTID + ret << m.COMMID + ret << m.COMMNAME + + def web = "" + if(k=="zhuge_avg_price"){ + web = "诸葛找房" + }else if(k=="anjuke_avg_price"){ + web = "安居客" + }else if(k=="fang_avg_price"){ + web = "房天下" + }else if(k=="lianjia_avg_price"){ + web = "链家" + }else if(k=="cityhouse_avg_price"){ + web = "禧泰数据" + }else if(k=="maitian_avg_price"){ + web = "麦田" + }else if(k=="ke_avg_price"){ + web = "贝壳" + }else if(k=="taofang_avg_price"){ + web = "淘房365" + }else if(k=="zhongyuan_avg_price"){ + web = "中原" + }else if(k=="tbs_avg_price"){ + web = "兔博士" + }else if(k=="qfang_avg_price"){ + web = "Q房" + }else if(k=="fangtan_avg_price"){ + web = "房探" + }else if(k=="fangduoduo_avg_price"){ + web = "房多多" + }else if(k=="wiwj_avg_price"){ + web = "我爱我家" + } + + ret << sourceMap.get(web) + ret << web + ret << m.YEAR_MONTH + ret << v + ret << "" + ret << "" + ret << m.updatetime + results << ret + } + } + + return results + + } + + + def findThirdPartyPrice(def city, def statMap){ + def q = new BasicDBObject() + q.put("city", city) + q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) + + def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1)) + def map = [:] + for(def db : dbcursor){ + def key = db.region+"##"+db.name + def dataMap = map.get(key) + if(!dataMap){ + dataMap = [:] + dataMap.put("id", db._id.toString()) + map.put(key, dataMap) + } + if(!dataMap.containsKey("zhugezhaofang_esf_price") && db.zhugezhaofang_esf_price && db.zhugezhaofang_esf_price>3000){ + dataMap.put("zhugezhaofang_esf_price", db.zhugezhaofang_esf_price) + } + if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){ + dataMap.put("anjuke_esf_price", db.anjuke_esf_price) + } + if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){ + dataMap.put("anjuke_esf_price", db.get("58tc_esf_price")) + } + if(!dataMap.containsKey("soufang_esf_price") && db.soufang_esf_price && db.soufang_esf_price>3000){ + dataMap.put("soufang_esf_price", db.soufang_esf_price) + } + if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){ + dataMap.put("lianjia_esf_price", db.lianjia_esf_price) + } + if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){ + dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price) + } + + if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price && db.maitian_esf_price>3000){ + dataMap.put("maitian_esf_price", db.maitian_esf_price) + } + + if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){ + dataMap.put("beike_esf_price", db.beike_esf_price) + } + + if(!dataMap.get("365house_esf_price") && db.get("365house_esf_price") && db.get("365house_esf_price")>3000){ + dataMap.put("365house_esf_price", db.get("365house_esf_price")) + } + + if(!dataMap.containsKey("zhongyuan_esf_price") && db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){ + dataMap.put("zhongyuan_esf_price", db.zhongyuan_esf_price) + } + + if(!dataMap.containsKey("tuboshi_esf_price") && db.tuboshi_esf_price && db.tuboshi_esf_price>3000){ + dataMap.put("tuboshi_esf_price", db.tuboshi_esf_price) + } + + if(!dataMap.containsKey("qfang_esf_price") && db.qfang_esf_price && db.qfang_esf_price>3000){ + dataMap.put("qfang_esf_price", db.qfang_esf_price) + } + + if(!dataMap.containsKey("fangtan_esf_price") && db.fangtan_esf_price && db.fangtan_esf_price>3000){ + dataMap.put("fangtan_esf_price", db.fangtan_esf_price) + } + + if(!dataMap.containsKey("fangduoduo_esf_price") && db.fangduoduo_esf_price && db.fangduoduo_esf_price>3000){ + dataMap.put("fangduoduo_esf_price", db.fangduoduo_esf_price) + } + + if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){ + dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price")) + } +// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){ +// map.put(key, db) +// } + } + + def resultMap = [:] + map.each {key, value-> + if(value && value.size()>1){ + if(value.size()==2 && !value.lianjia_esf_price && !value.beike_esf_price){ + def price = statMap.get(key) + if(price){ + def priceMap = [:] + def startPrice = (int)(price - price*0.3) + def endPrice = (int)(price*0.3 + price) + value.each{pk,pv-> + if(pk.contains("price")){ + if(pv>=startPrice && pv<=endPrice){ + priceMap.put(pk,pv) + } + }else{ + priceMap.put(pk,pv) + } + } + if(priceMap.size()>1){ + resultMap.put(key, priceMap) + } + } + }else{ + resultMap.put(key, value) + } + + } + } + return resultMap + } + + + def getDistrictStatByCity(city, weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("weekend", weekend) + def dbcursor = DIST_SELL_STAT.find(q) + def map = new ConcurrentHashMap() + for(def d : dbcursor){ + map.put(d.region+"##"+d.district_name, d.avg_price) + } + return map + } + + + def saveToPgSql(rows){ + for(def r : rows) { + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ + r[i] = null + } + } + r << null + r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28" + r << null + r << '2999-12-31' + r << null + r << null + r << null + } + + println "start: "+new Date() + def sql = "insert into base.bigdata_third_party_price values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + pgsqlInstance.withBatch(50000, sql){ ps-> + for(def r : rows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18]) + } + } + + println "end: "+new Date() + } + + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy index 81977584c..29214f73f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy @@ -33,11 +33,11 @@ class AddPriceTest { task.run(); } def run(){ - def filePath = "D:\\数据\\建行\\房产估值\\指导价\\补充154价格.xlsx" + def filePath = "D:\\数据\\建行\\对标补充数据.xlsx" // addGXDPrice(filePath) // addCcbPrice(filePath) -// addThirdPrice(filePath) - addFjwPrice(filePath) + addThirdPrice(filePath) +// addFjwPrice(filePath) // exportPkPrice() // addRegion() @@ -190,11 +190,11 @@ class AddPriceTest { def addThirdPrice(filepath){ ExcelUtils.multiThreadingModify(filepath, { d -> - if (d['M']) { - def dist = CommonDatautils.findDistrictById(d['M']) + if (d['G']) { + def dist = CommonDatautils.findDistrictById(d['G']) if(dist){ def thirdPrice = findThirdPartyPrice(dist.city, dist.region, dist.name) - d['AN'] = thirdPrice + d['Z'] = thirdPrice } } }) @@ -206,7 +206,7 @@ class AddPriceTest { q.put("city", city) q.put("region", region) q.put("name", name) - q.put("s_date", new BasicDBObject('$gte', 202212).append('$lte',202302)) + q.put("s_date", new BasicDBObject('$gte', 202311).append('$lte',202404)) def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1)) def price = null @@ -216,10 +216,10 @@ class AddPriceTest { break; } - if(!price && db.beike_esf_price && db.beike_esf_price>2000){ - price = db.beike_esf_price - break - } +// if(!price && db.beike_esf_price && db.beike_esf_price>2000){ +// price = db.beike_esf_price +// break +// } } return price } @@ -256,14 +256,15 @@ class AddPriceTest { def addFjwPrice(path){ ExcelUtils.multiThreadingModify(path,{d-> - if(!d['F']){ - if(d['E'] instanceof String){ - def dist = CommonDatautils.findDistrictById(d['E']) + if(!d['W']){ + if(d['G'] instanceof String){ + def dist = CommonDatautils.findDistrictById(d['G']) if(dist){ // d['N'] = dist.region def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name) if(dbo){ - d['F'] = dbo.avg_price + d['W'] = dbo.avg_price + d['X'] = dbo.cal_type // d['J'] = dbo.avg_price_2b } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy index 9c6952a70..aeda5fc28 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy @@ -29,11 +29,11 @@ import static com.mongodb.client.model.Sorts.* public class DbUtils{ public static final String ONLINE_MONGODB_DOMAIN = "mongo.db.fangjia.com"; - public static DBCollection OPERATION_LOG = new Mongo("192.168.0.136", 27017).getDB('fangjia').getCollection('operation_log'); + //public static DBCollection OPERATION_LOG = new Mongo("192.168.0.136", 27017).getDB('fangjia').getCollection('operation_log'); public static final String COMMON_DB = "fangjia"; public static final int COMMON_PORT = 27017; - public static DSCollection seawater = new DataStore("seawater.db.fangjia.com", 27777).getCollection("fangjia.seawater") + //public static DSCollection seawater = new DataStore("seawater.db.fangjia.com", 27777).getCollection("fangjia.seawater") public static DSCollection seaweed = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed") public static DSCollection SEAWEED_SLAVE = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed") public static DSCollection seabird = new DataStore("seabird.db.fangjia.com").getCollection("fangjia.seabird")