diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 03bc56f0a..39b653a2f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -32,7 +32,7 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataDealBatch extends BaseBatch{ - def curDate = 20211001//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); + def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"] @@ -54,9 +54,9 @@ class BigdataDealBatch extends BaseBatch{ //def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date()) def citys = CcbCommonUtils.citys; -// for(def city : citys){ -// addDealPriceMatch(city); -// } + for(def city : citys){ + addDealPriceMatch(city); + } println citys.unique().size() def rows = [] def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] @@ -443,15 +443,9 @@ class BigdataDealBatch extends BaseBatch{ // def citys = ["北京","广州","深圳","重庆","郑州","西安","成都","东莞","佛山","杭州","合肥","南京","长春","扬州","无锡","太原","台州","泉州","宁波","南通","南昌","贵阳","哈尔滨","惠州","济南","金华","昆明","遵义","镇江","肇庆","宜昌","芜湖","泰州","汕头","清远","秦皇岛","马鞍山","洛阳","连云港","九江","淮安","呼和浩特","沧州","保定","安庆"] - def startDate = null; -// if(citys.contains(city)){ - startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) -// }else{ -// startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) -// } + def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date()) -// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210801", "yyyyMMdd")) for(def i=0;i m.deal_price){ + newPrice = m.deal_price*precent + m.deal_price + }else if(m.unit_price < m.deal_price){ + newPrice = m.deal_price - (m.deal_price*precent) + } + if(newPrice){ + updateDealPrice(m, newPrice.toInteger()) + } + } + } + +// if(d['M']){ +// def rets = findThirdByPe(d['A'], yearMonth, d['M']) +// for(def r : rets){ +// deletThirdPrice(r, thirdMonth) +// } +// } + } + } void process() { @@ -96,13 +138,18 @@ class DealPriceDataProcess { def deleteDealPrice(r){ def sql = "delete from bigdata_deal_gj where commid='${r.commid}' and buildarea=${r.buildarea} and DEALUNITPRICE=${r.deal_price} " + "and dealdate=to_date('${r.dealdate}','yyyy-mm-dd')" - sqlInstance.executeUpdate(sql) + sqlInstance236.executeUpdate(sql) + } + + def deletThirdPrice(r, thirdMonth){ + def sql = "delete from bigdata_third_party_price_gj where price_date=${thirdMonth} and commid='${r.commid}' and community_price=${r.target_avg_price}" + sqlInstance236.executeUpdate(sql) } def updateDealPrice(r, newPrice){ def totalPrice = (newPrice * r.buildarea).toInteger() def sql = "update bigdata_deal_gj set DEALUNITPRICE=${newPrice},DEALTOTALPRICE=${totalPrice} where commid='${r.commid}' and buildarea=${r.buildarea} and DEALUNITPRICE=${r.deal_price} and dealdate=to_date('${r.dealdate}','yyyy-mm-dd')" - sqlInstance.executeUpdate(sql) + sqlInstance236.executeUpdate(sql) } def findLtePPeCount(city, year_month){ @@ -114,6 +161,39 @@ class DealPriceDataProcess { return count } + def findDealByPe(city, yearMonth, peVal){ + def sql = "select cityid,cityname,distid,commid,commname,unit_price,target_avg_price,pe,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate from avm_evaluation_item_gj where year_month=${yearMonth} and cityname='${city}' and data_type='成交' and pe>${peVal}"; + def rets = [] + sqlInstance236.eachRow(sql,{r-> + def map = [:] + map.cityname = r.cityname + map.cityid=r.cityid + map.distid = r.distid + map.commid = r.commid + map.commname = r.commname + map.deal_price = r.target_avg_price + map.unit_price = r.unit_price + map.buildarea = r.buildarea + map.ppe = r.pe + map.dealdate = r.dealdate + rets << map + }) + return rets + } + + def findThirdByPe(city, yearMonth, peVal){ + def sql = "select year_month, commid, target_avg_price from avm_evaluation_item_gj where year_month=${yearMonth} and cityname='${city}' and data_type='第三方' and pe>${peVal}"; + def rets = [] + sqlInstance236.eachRow(sql,{r-> + def map = [:] + map.year_month = r.year_month + map.commid = r.commid + map.target_avg_price = r.target_avg_price + rets << map + }) + return rets + } + def findAllGtePPeDatas(city, yearMonth){ def sql = "select cityid,cityname,distid,commid,commname,unit_price,target_avg_price,pe,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate from avm_evaluation_item_gj where year_month=${yearMonth} and cityname='${city}' and data_type='成交' and pe>15 order by pe asc"; @@ -147,6 +227,27 @@ class DealPriceDataProcess { }); return map } + + + def findCityByPpeRange(def cityname, def yearMonth, ppeStart, ppeEnd){ + def sql = "select cityid,cityname,distid,commid,commname,unit_price,target_avg_price,pe,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate from avm_evaluation_item_gj where cityname='${cityname}' and year_month=${yearMonth} and data_type='成交' and pe>${ppeStart} and pe<${ppeEnd}"; + def rets = [] + sqlInstance236.eachRow(sql,{r-> + def map = [:] + map.cityname = r.cityname + map.cityid=r.cityid + map.distid = r.distid + map.commid = r.commid + map.commname = r.commname + map.deal_price = r.target_avg_price + map.unit_price = r.unit_price + map.ppe = r.pe + map.buildarea = r.buildarea + map.dealdate = r.dealdate + rets << map + }); + return rets + } def getCityPPeIndex(def filePath){ def datas = ExcelUtils.read(filePath,0,1) def map = [:] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index d6ba74db8..4e28e77d6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -11,7 +11,7 @@ import java.sql.ResultSet class ExportOracleData { - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") @@ -24,30 +24,36 @@ class ExportOracleData { } def run() { + +// exportGuidedPrice() +// exportCommunityGuidedPrice() +// +// //exportThirdPartyPrice()//废弃掉,老表 + exportThirdPartyPrice2(202111) +// +// +// +// exportBigdataForsale("2021-11-19") +// // exportEvaluationRateData() // exportBigdataPoiData() // exportCommunityStaging() // exportCommunityBuileType() // exportBigdataDistrictData() // exportBigdataCommunity() -// exportBigdataDeal("2021-10-01") + exportBigdataDeal("2021-11-01") -// exportBigdataForsale("2021-10-19") - exportGuidedPrice() - exportCommunityGuidedPrice() - -// exportThirdPartyPrice()//废弃掉,老表 -// exportThirdPartyPrice2() - - - exportAvmEvaluationGJNB(202111) - exportAVMEvaluationItemGJNB(202111) - exportCommunityBasePrice(202111) +// exportAvmEvaluationGJNB(202111) +// exportAVMEvaluationItemGJNB(202111) +// exportCommunityBasePrice(202111) // exportCityPrice(202111) // exportRegionPrice(202111) // exportCommunityBasePriceGj(202111) + +// exportCommunityPdescribeGj(202112) +// exportCommCaseDetailGj(202112) // exportLastFile() } @@ -422,8 +428,8 @@ class ExportOracleData { } - def exportThirdPartyPrice2() { - def sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ"; + def exportThirdPartyPrice2(price_date) { + def sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where price_date=${price_date} "; def allRows = [] sqlInstance.eachRow(sql, { r -> def data = [] @@ -444,8 +450,8 @@ class ExportOracleData { def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb/MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { -// fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" - fileName = "D:\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" +// fileName = "D:\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" } def headers = [] //["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] @@ -858,6 +864,114 @@ class ExportOracleData { println "DISTRICT_PRICE_GJ 上传Ftp完成......" } + def exportCommunityPdescribeGj(month){ + + def citys = CcbCommonUtils.citys; + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] + + for(def city : citys) { + def cityCode = cityCodeDataMap.get(city) + def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month} and cityid='${cityCode}'"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMUNITY_ID + data << r.CITYID + data << r.CITY_NAME + data << r.DISTRICT_CODE + data << r.DISTRICT_NAME + data << r.COMMUNITY_NAME + data << r.PQ_NAME + data << r.BASE_PRICE + data << r.CONFIDENCE_LEVEL + data << r.CNT + data << r.MIN_PRICE + data << r.MEDIANPRICE + data << r.MEANPRICE + data << r.MAX_PRICE + data << r.TIME + data << r.FIRSTDATAT + data << r.THIRDDATA + data << r.RELATIVEERROR + data << r.SKEWNESS + data << r.PQ_PRICE_RANGE + data << r.DISTRICT_PRICE_RANGE + data << r.DESCRIPTION + data << r.PRIV_BASE_PRICE + data << r.IS_ARTTFICIAL + data << r.PRIV_FINAL_PRICE + data << r.CREATE_DATE + data << r.PRICE + data << r.PRICE_SOURCE + data << r.CASE_MD_PRICE + data << r.CASE_AVG_PRICE + data << r.INDEX_LABEL + data << r.MAIN_PROPERTY + data << r.COMMTYPE + data << r.NEW_DEAL_PRICE + data << r.NEW_DEAL_DATE + data << r.PRICE_MOM_RATIO + data << r.MAX_MOM_RATIO + data << r.PRICE_YOY_RATIO + data << r.SATUR_RATIO + data << r.NEIGHBOR_PRICE + allRows << data + }); + println city + " "+allRows.size() + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + } + println "community_p_describe_gj 数据文件输出完成......" + CcbFtpUtils.ftpUpload(fileName, curDate) + println "community_p_describe_gj 上传Ftp完成......" + } + + def exportCommCaseDetailGj(month){ + + def citys = CcbCommonUtils.citys; + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def fileName = "/ccb/MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] + + for(def city : citys){ + def cityCode = cityCodeDataMap.get(city) + def sql = "select ID, CITYID, COMMUNITY_ID, NETLISTING_ID, PRICE_DATE, to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE from comm_case_detail_gj where cityid='${cityCode}'";//price_date=${month} and + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.COMMUNITY_ID + data << r.NETLISTING_ID + data << r.PRICE_DATE + data << r.CREATE_DATE + allRows << data + if(allRows.size()%200000==0){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + allRows.clear() + } + }); + if(allRows.size()>0){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + } + } + + println "comm_case_detail_gj 数据文件输出完成......" + CcbFtpUtils.ftpUpload(fileName, curDate) + println "comm_case_detail_gj 上传Ftp完成......" + } + def exportLastFile(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def dataCodeFileName = "/ccb/${curDate}.ok" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 0564c1f41..9450808e8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -30,6 +30,9 @@ import java.util.regex.*; def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") +def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") @@ -221,18 +224,18 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def } -//def fileName = "D:\\数据\\建行\\建行成交\\成交数据_1025 - 副本.xlsx" -//def dealMap = findDealCount(sqlInstance, "2021-10-01", "2021-11-01") -//def testMap = findCaluateCityPPE(sqlInstance94, 202111) -//ExcelUtils.modify(fileName,{d-> -// if(d['C']){ -// def oldcount = dealMap.get(d['C']) -// d['N'] = oldcount -// def casecount = testMap.get(d['C']) -// d['O'] = casecount -// println d -// } -//},0) +def fileName = "D:\\数据\\建行\\房产估值\\成交数据_1222 - 副本.xlsx" +def dealMap = findDealCount(sqlInstance236, "2021-11-01", "2021-12-01") +def testMap = findCaluateCityPPE(sqlInstance236, 202112) +ExcelUtils.modify(fileName,{d-> + if(d['C']){ + def oldcount = dealMap.get(d['C']) + d['P'] = oldcount + def casecount = testMap.get(d['C']) + d['Q'] = casecount + println d + } +},0) //ExcelUtils.multiThreadingModify("D:\\数据\\小区\\5个城市小区列表0723(1).xlsx",{d-> // if(d['I'] && d['K']){ @@ -300,79 +303,79 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def //},0) -def index = 0 -ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\重庆反查数据_669(审核完毕).xlsx",{d-> - if(d['A']){ - def dist = CommonDatautils.findDistrictById(d['A']) - if(dist){ - - def alias = [] - if(d['H'] && d['H']!=dist.name){ - alias << d['H'] - if(d['H'].contains("·")){ - alias << d['H'].replace("·", "") - } - if(dist.alias){ - for(def aliasName : dist.alias){ - if(!alias.contains(aliasName)){ - alias << aliasName - } - } - } - } - - def address = null - if(d['I']){ - address = AddressUtil.formatAddress(dist.city, d['I']) - } - - Long lng = null - Long lat = null - Long lat2 = null - Long lng2 = null - if(d['J']){ - lng = d['J'].toString().toDouble() * 10000000000l; - lat = d['K'].toString().toDouble() * 10000000000l; - def json = BaiDuApiUtils.geoconv(d['J'].toString().toDouble(), d['K'].toString().toDouble(), 3) - if(json != null && json.containsKey("x")) { - lat2 = json.get("y") * 10000000000l - lng2 = json.get("x") * 10000000000l - } - } - - def region = null - if(dist.region!=d['F']){ - region = dist.region - } - - def udata = new BasicDBObject() - if(region){ - udata.put("region", region) - } - - if(address){ - udata.put("address", address) - } - if(lat2){ - udata.put("lng", lng) - udata.put("lat", lat) - udata.put("lng2", lng2) - udata.put("lat2", lat2) - } - - if(alias){ - udata.put("alias", alias) - } - if(udata){ - index++ - DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) - println index+ " "+dist.city +" "+dist.name+" "+udata - } - - } - } - -},0) +//def index = 0 +//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\重庆反查数据_669(审核完毕).xlsx",{d-> +// if(d['A']){ +// def dist = CommonDatautils.findDistrictById(d['A']) +// if(dist){ +// +// def alias = [] +// if(d['H'] && d['H']!=dist.name){ +// alias << d['H'] +// if(d['H'].contains("·")){ +// alias << d['H'].replace("·", "") +// } +// if(dist.alias){ +// for(def aliasName : dist.alias){ +// if(!alias.contains(aliasName)){ +// alias << aliasName +// } +// } +// } +// } +// +// def address = null +// if(d['I']){ +// address = AddressUtil.formatAddress(dist.city, d['I']) +// } +// +// Long lng = null +// Long lat = null +// Long lat2 = null +// Long lng2 = null +// if(d['J']){ +// lng = d['J'].toString().toDouble() * 10000000000l; +// lat = d['K'].toString().toDouble() * 10000000000l; +// def json = BaiDuApiUtils.geoconv(d['J'].toString().toDouble(), d['K'].toString().toDouble(), 3) +// if(json != null && json.containsKey("x")) { +// lat2 = json.get("y") * 10000000000l +// lng2 = json.get("x") * 10000000000l +// } +// } +// +// def region = null +// if(dist.region!=d['F']){ +// region = dist.region +// } +// +// def udata = new BasicDBObject() +// if(region){ +// udata.put("region", region) +// } +// +// if(address){ +// udata.put("address", address) +// } +// if(lat2){ +// udata.put("lng", lng) +// udata.put("lat", lat) +// udata.put("lng2", lng2) +// udata.put("lat2", lat2) +// } +// +// if(alias){ +// udata.put("alias", alias) +// } +// if(udata){ +// index++ +// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) +// println index+ " "+dist.city +" "+dist.name+" "+udata +// } +// +// } +// } +// +//},0) @@ -440,7 +443,7 @@ ExcelUtils.multiThreadingModify("D:\\ def findDealCount(sqlInstance, def startDate, def endDate){ - def sql = "select cityname,count(*) as count from bigdata_deal_gj_test where dealdate>=to_date('${startDate}','yyyy-mm-dd') and dealdate + def lastYear = DateUtils.add(DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd"), -365) + def lastWeekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightYYYYMMDD(lastYear)) + def districtPriceMap = CommonDatautils.queryAllDistrictPriceByWeekend(city, lastWeekend) + + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\租售比\\"+f+".xlsx",{d-> if(d['E']){ def region = d['E'] def newRegion = RegionData.formatRegion(city, region) @@ -148,12 +152,34 @@ class MatchMapData { d['K']="映射别名" d['L'] = dist.name_cat }else{ - dist = CommonDatautils.findDistrictByAlias(city, newRegion, d['F'],"district") - if(dist){ + def rets = CommonDatautils.findDistrictByAlias(city, newRegion, d['F'],"district") + if(rets){ d['K']="别名" } + if(rets){ + dist = rets[0] + } } } + + if(dist){ + def nearDbo = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name) + + def lastMonth = DateUtils.parseFromTightYYYYMMDD("2021-11-30") + def sqmetre = CommonDatautils.findSqmetreRent(dist.city, dist.region, dist.name, lastMonth) + if(nearDbo && districtPriceMap){ + def priceObj = districtPriceMap.get(dist.region+"#"+dist.name) + if(priceObj){ + def markup = (nearDbo.avg_price - priceObj.avg_price)*100/priceObj.avg_price + d['M'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, markup) + } + } + if(nearDbo && sqmetre){ + def prc = nearDbo.avg_price/sqmetre.avg_price + d['N'] = prc.toInteger() + } + d['O'] = dist.estate_type2 + } }else{ d['M'] = "区域无法标准化" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy index 599442639..b25c8bc10 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy @@ -32,7 +32,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") public static def batchsize = 1000 @@ -42,7 +42,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { @Override public void process(Object args) { - def citys = ["苏州","北京","广州","杭州","合肥","深圳","石家庄","武汉","西安","重庆"]//CcbCommonUtils.citys; + def citys = CcbCommonUtils.citys; def rows = [] def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() // for (def city : citys) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 596a0c6a6..ec499879a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -16,7 +16,7 @@ import java.sql.ResultSetMetaData @BatchOwner(Watcher.GaoJuan) class ExportDistrictRongHe extends BaseBatch{ - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") static main(args) { runBatch(new ExportDistrictRongHe(), args); @@ -31,20 +31,17 @@ class ExportDistrictRongHe extends BaseBatch{ // exportAllBigdataDealFJW() for(def city : citys) { def cityCode = cityCodeDataMap.get(city) -//// if(!codes.contains(cityCode)){ -//// println city +" "+cityCode -//// } - exportBigdataCommunityGJRH(cityCode) - exportBigdataBuildingFJW(cityCode) - exportBigdataCommfeeGj(cityCode) +// exportBigdataCommunityGJRH(cityCode) +// exportBigdataBuildingFJW(cityCode) +// exportBigdataCommfeeGj(cityCode) // exportBigdataDealFJW(cityCode) - exportBigdataCommbuildstruGJ(cityCode) - exportBigdataCommhousetypeGJ(cityCode) - exportBigdataCommbuildyearGj(cityCode) - exportBigdataCommhouseuseGJ(cityCode) - exportBigdataPositionGJ(cityCode) - exportBigdataDistrictFJW(cityCode) +// exportBigdataCommbuildstruGJ(cityCode) +// exportBigdataCommhousetypeGJ(cityCode) +// exportBigdataCommbuildyearGj(cityCode) +// exportBigdataCommhouseuseGJ(cityCode) +// exportBigdataPositionGJ(cityCode) +// exportBigdataDistrictFJW(cityCode) exportBigdataAliasFJW(cityCode) // // exportBigdataBuildprojFjw(cityCode) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 1549f70c3..ee7114640 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -66,6 +66,8 @@ class CommonDatautils { public static final DBCollection POI_COLL = MongoUtils.getCollectionByName2("fangjia_base", "baidu_place_poi", mongoDbUrl) public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", mongoDbUrl) + public static final DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName2("fangjia", "sqmetre_rent", mongoDbUrl) + public static DSCollection getHouseDataStoreConnection(){ if(house_dataStoreConnection != null) { return house_dataStoreConnection @@ -3097,5 +3099,19 @@ class CommonDatautils { def count = DEAL_PRICE_COLL.count(q); return count; } + + static def findSqmetreRent(def city, def region, def name, def weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("district_name", name) + q.put("weekend", weekend) + def dbcur = SQMETRE_RENT_COLL.find(q) + def dbo = null + for(def d : dbcur){ + dbo = d; + } + return dbo; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index e05d00dfa..a1350cc2f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -48,7 +48,7 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2021-10-01" + static def statDate = "2021-11-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo @@ -285,7 +285,7 @@ class YouDaDataParser { def esfParser(){ - def files = ["D:\\数据\\友达\\2021年11月\\二手房\\查询1.DBF"] + def files = ["D:\\数据\\友达\\2021年12月\\二手房\\查询1.DBF"] def datas = [] for(def fileName : files){ def rows = readDBF(fileName); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index 3317161c0..c160a713a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -38,8 +38,8 @@ class Esf58Crawl extends BaseBatch{ def static useDistKeys = [] static def totalCount = new AtomicLong() //"福州":"fz|304","合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319","成都":"cd|102","上海":"sh|2", "北京":"bj|1","天津":"tj|18", - //"重庆":"cq|37","石家庄":"sjz|241","广州":"gz|3","深圳":"sz|4", - def cityList = ["东莞":"dg|413","佛山":"fs|222","杭州":"hz|79","武汉":"wh|158", + //"重庆":"cq|37","石家庄":"sjz|241","广州":"gz|3","深圳":"sz|4","东莞":"dg|413","佛山":"fs|222","杭州":"hz|79","武汉":"wh|158", + def cityList = [ "青岛":"qd|122","南京":"nj|172","嘉兴":"jx|497","大连":"dl|147","乌鲁木齐":"xj|984","太原":"ty|740","兰州":"lz|952","南昌":"nc|669", "沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362","常州":"cz|463", "南通":"nt|394","吉林":"jl|700","中山":"zs|771","台州":"tz|403","东营":"dy|623","三亚":"sanya|2422", "呼和浩特":"hu|811","济南":"jn|265","温州":"wz|330","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096",