This commit is contained in:
2021-12-24 15:52:16 +08:00
parent baf4c7c361
commit 516ea36f15
14 changed files with 406 additions and 157 deletions
@@ -32,7 +32,7 @@ import java.util.regex.Pattern
@BatchOwner(Watcher.GaoJuan)
class BigdataDealBatch extends BaseBatch{
def curDate = 20211001//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
@@ -54,9 +54,9 @@ class BigdataDealBatch extends BaseBatch{
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
def citys = CcbCommonUtils.citys;
// for(def city : citys){
// addDealPriceMatch(city);
// }
for(def city : citys){
addDealPriceMatch(city);
}
println citys.unique().size()
def rows = []
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
@@ -443,15 +443,9 @@ class BigdataDealBatch extends BaseBatch{
// def citys = ["北京","广州","深圳","重庆","郑州","西安","成都","东莞","佛山","杭州","合肥","南京","长春","扬州","无锡","太原","台州","泉州","宁波","南通","南昌","贵阳","哈尔滨","惠州","济南","金华","昆明","遵义","镇江","肇庆","宜昌","芜湖","泰州","汕头","清远","秦皇岛","马鞍山","洛阳","连云港","九江","淮安","呼和浩特","沧州","保定","安庆"]
def startDate = null;
// if(citys.contains(city)){
startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// }else{
// startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// }
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210801", "yyyyMMdd"))
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
fixedThreadPool.execute(new Runnable() {
@@ -511,11 +505,9 @@ class BigdataDealBatch extends BaseBatch{
def getYouDaData(statMap){
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210801", "yyyyMMdd"))
def q = new BasicDBObject()
q.put("contractTime", new BasicDBObject('$gte', startDate).append('$lte', endDate));
q.put("area", new BasicDBObject('$gte', 20))
// q.put("premisesName" , "呼玛二村")
def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000)
def rets = []
@@ -397,7 +397,7 @@ class BigdataForsaleBatch extends BaseBatch{
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
@@ -35,7 +35,7 @@ class CcbCommonUtils {
public static DSCollection dataStoreConnection = null;
public static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
public static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
public static def findCityCodeValue(){
@@ -19,11 +19,53 @@ class DealPriceDataProcess {
def sqlInstance2 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new DealPriceDataProcess()
// task.process()//当成交pe值不好时运行,拉升成交价格及删除pe太大数据
// task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据
// task.addLianJiaPrice();
task.deleteThirdPartyData()
// task.deleteThirdPartyData()
task.delDealByPe()
}
def delDealByPe(){
def yearMonth = 202112
def thirdMonth = 202111
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\估值模型测试202112_1223 - 副本.xlsx",0,1)
for(def d : datas){
// if(d['G']){
// def rets = findDealByPe(d['A'], yearMonth, d['G'])
// for(def r : rets){
// deleteDealPrice(r)
// }
// }
if(d['H']){
def rets = findCityByPpeRange(d['A'], yearMonth, 15, 20)
for(def m : rets){
def precent = d['H']/100.0
def newPrice = null
if(m.unit_price > m.deal_price){
newPrice = m.deal_price*precent + m.deal_price
}else if(m.unit_price < m.deal_price){
newPrice = m.deal_price - (m.deal_price*precent)
}
if(newPrice){
updateDealPrice(m, newPrice.toInteger())
}
}
}
// if(d['M']){
// def rets = findThirdByPe(d['A'], yearMonth, d['M'])
// for(def r : rets){
// deletThirdPrice(r, thirdMonth)
// }
// }
}
}
void process() {
@@ -96,13 +138,18 @@ class DealPriceDataProcess {
def deleteDealPrice(r){
def sql = "delete from bigdata_deal_gj where commid='${r.commid}' and buildarea=${r.buildarea} and DEALUNITPRICE=${r.deal_price} " +
"and dealdate=to_date('${r.dealdate}','yyyy-mm-dd')"
sqlInstance.executeUpdate(sql)
sqlInstance236.executeUpdate(sql)
}
def deletThirdPrice(r, thirdMonth){
def sql = "delete from bigdata_third_party_price_gj where price_date=${thirdMonth} and commid='${r.commid}' and community_price=${r.target_avg_price}"
sqlInstance236.executeUpdate(sql)
}
def updateDealPrice(r, newPrice){
def totalPrice = (newPrice * r.buildarea).toInteger()
def sql = "update bigdata_deal_gj set DEALUNITPRICE=${newPrice},DEALTOTALPRICE=${totalPrice} where commid='${r.commid}' and buildarea=${r.buildarea} and DEALUNITPRICE=${r.deal_price} and dealdate=to_date('${r.dealdate}','yyyy-mm-dd')"
sqlInstance.executeUpdate(sql)
sqlInstance236.executeUpdate(sql)
}
def findLtePPeCount(city, year_month){
@@ -114,6 +161,39 @@ class DealPriceDataProcess {
return count
}
def findDealByPe(city, yearMonth, peVal){
def sql = "select cityid,cityname,distid,commid,commname,unit_price,target_avg_price,pe,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate from avm_evaluation_item_gj where year_month=${yearMonth} and cityname='${city}' and data_type='成交' and pe>${peVal}";
def rets = []
sqlInstance236.eachRow(sql,{r->
def map = [:]
map.cityname = r.cityname
map.cityid=r.cityid
map.distid = r.distid
map.commid = r.commid
map.commname = r.commname
map.deal_price = r.target_avg_price
map.unit_price = r.unit_price
map.buildarea = r.buildarea
map.ppe = r.pe
map.dealdate = r.dealdate
rets << map
})
return rets
}
def findThirdByPe(city, yearMonth, peVal){
def sql = "select year_month, commid, target_avg_price from avm_evaluation_item_gj where year_month=${yearMonth} and cityname='${city}' and data_type='第三方' and pe>${peVal}";
def rets = []
sqlInstance236.eachRow(sql,{r->
def map = [:]
map.year_month = r.year_month
map.commid = r.commid
map.target_avg_price = r.target_avg_price
rets << map
})
return rets
}
def findAllGtePPeDatas(city, yearMonth){
def sql = "select cityid,cityname,distid,commid,commname,unit_price,target_avg_price,pe,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate from avm_evaluation_item_gj where year_month=${yearMonth} and cityname='${city}' and data_type='成交' and pe>15 order by pe asc";
@@ -147,6 +227,27 @@ class DealPriceDataProcess {
});
return map
}
def findCityByPpeRange(def cityname, def yearMonth, ppeStart, ppeEnd){
def sql = "select cityid,cityname,distid,commid,commname,unit_price,target_avg_price,pe,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate from avm_evaluation_item_gj where cityname='${cityname}' and year_month=${yearMonth} and data_type='成交' and pe>${ppeStart} and pe<${ppeEnd}";
def rets = []
sqlInstance236.eachRow(sql,{r->
def map = [:]
map.cityname = r.cityname
map.cityid=r.cityid
map.distid = r.distid
map.commid = r.commid
map.commname = r.commname
map.deal_price = r.target_avg_price
map.unit_price = r.unit_price
map.ppe = r.pe
map.buildarea = r.buildarea
map.dealdate = r.dealdate
rets << map
});
return rets
}
def getCityPPeIndex(def filePath){
def datas = ExcelUtils.read(filePath,0,1)
def map = [:]
@@ -11,7 +11,7 @@ import java.sql.ResultSet
class ExportOracleData {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
@@ -24,30 +24,36 @@ class ExportOracleData {
}
def run() {
// exportGuidedPrice()
// exportCommunityGuidedPrice()
//
// //exportThirdPartyPrice()//废弃掉,老表
exportThirdPartyPrice2(202111)
//
//
//
// exportBigdataForsale("2021-11-19")
//
// exportEvaluationRateData()
// exportBigdataPoiData()
// exportCommunityStaging()
// exportCommunityBuileType()
// exportBigdataDistrictData()
// exportBigdataCommunity()
// exportBigdataDeal("2021-10-01")
exportBigdataDeal("2021-11-01")
// exportBigdataForsale("2021-10-19")
exportGuidedPrice()
exportCommunityGuidedPrice()
// exportThirdPartyPrice()//废弃掉,老表
// exportThirdPartyPrice2()
exportAvmEvaluationGJNB(202111)
exportAVMEvaluationItemGJNB(202111)
exportCommunityBasePrice(202111)
// exportAvmEvaluationGJNB(202111)
// exportAVMEvaluationItemGJNB(202111)
// exportCommunityBasePrice(202111)
// exportCityPrice(202111)
// exportRegionPrice(202111)
// exportCommunityBasePriceGj(202111)
// exportCommunityPdescribeGj(202112)
// exportCommCaseDetailGj(202112)
// exportLastFile()
}
@@ -422,8 +428,8 @@ class ExportOracleData {
}
def exportThirdPartyPrice2() {
def sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ";
def exportThirdPartyPrice2(price_date) {
def sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where price_date=${price_date} ";
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
@@ -444,8 +450,8 @@ class ExportOracleData {
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
// fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
fileName = "D:\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
// fileName = "D:\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []
//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
@@ -858,6 +864,114 @@ class ExportOracleData {
println "DISTRICT_PRICE_GJ 上传Ftp完成......"
}
def exportCommunityPdescribeGj(month){
def citys = CcbCommonUtils.citys;
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []
for(def city : citys) {
def cityCode = cityCodeDataMap.get(city)
def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month} and cityid='${cityCode}'";
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.COMMUNITY_ID
data << r.CITYID
data << r.CITY_NAME
data << r.DISTRICT_CODE
data << r.DISTRICT_NAME
data << r.COMMUNITY_NAME
data << r.PQ_NAME
data << r.BASE_PRICE
data << r.CONFIDENCE_LEVEL
data << r.CNT
data << r.MIN_PRICE
data << r.MEDIANPRICE
data << r.MEANPRICE
data << r.MAX_PRICE
data << r.TIME
data << r.FIRSTDATAT
data << r.THIRDDATA
data << r.RELATIVEERROR
data << r.SKEWNESS
data << r.PQ_PRICE_RANGE
data << r.DISTRICT_PRICE_RANGE
data << r.DESCRIPTION
data << r.PRIV_BASE_PRICE
data << r.IS_ARTTFICIAL
data << r.PRIV_FINAL_PRICE
data << r.CREATE_DATE
data << r.PRICE
data << r.PRICE_SOURCE
data << r.CASE_MD_PRICE
data << r.CASE_AVG_PRICE
data << r.INDEX_LABEL
data << r.MAIN_PROPERTY
data << r.COMMTYPE
data << r.NEW_DEAL_PRICE
data << r.NEW_DEAL_DATE
data << r.PRICE_MOM_RATIO
data << r.MAX_MOM_RATIO
data << r.PRICE_YOY_RATIO
data << r.SATUR_RATIO
data << r.NEIGHBOR_PRICE
allRows << data
});
println city + " "+allRows.size()
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
}
println "community_p_describe_gj 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, curDate)
println "community_p_describe_gj 上传Ftp完成......"
}
def exportCommCaseDetailGj(month){
def citys = CcbCommonUtils.citys;
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb/MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat"
}
def headers = []
for(def city : citys){
def cityCode = cityCodeDataMap.get(city)
def sql = "select ID, CITYID, COMMUNITY_ID, NETLISTING_ID, PRICE_DATE, to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE from comm_case_detail_gj where cityid='${cityCode}'";//price_date=${month} and
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.COMMUNITY_ID
data << r.NETLISTING_ID
data << r.PRICE_DATE
data << r.CREATE_DATE
allRows << data
if(allRows.size()%200000==0){
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
allRows.clear()
}
});
if(allRows.size()>0){
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
}
}
println "comm_case_detail_gj 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, curDate)
println "comm_case_detail_gj 上传Ftp完成......"
}
def exportLastFile(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def dataCodeFileName = "/ccb/${curDate}.ok"
@@ -30,6 +30,9 @@ import java.util.regex.*;
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
@@ -221,18 +224,18 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
}
//def fileName = "D:\\数据\\建行\\建行成交\\成交数据_1025 - 副本.xlsx"
//def dealMap = findDealCount(sqlInstance, "2021-10-01", "2021-11-01")
//def testMap = findCaluateCityPPE(sqlInstance94, 202111)
//ExcelUtils.modify(fileName,{d->
// if(d['C']){
// def oldcount = dealMap.get(d['C'])
// d['N'] = oldcount
// def casecount = testMap.get(d['C'])
// d['O'] = casecount
// println d
// }
//},0)
def fileName = "D:\\数据\\建行\\房产估值\\成交数据_1222 - 副本.xlsx"
def dealMap = findDealCount(sqlInstance236, "2021-11-01", "2021-12-01")
def testMap = findCaluateCityPPE(sqlInstance236, 202112)
ExcelUtils.modify(fileName,{d->
if(d['C']){
def oldcount = dealMap.get(d['C'])
d['P'] = oldcount
def casecount = testMap.get(d['C'])
d['Q'] = casecount
println d
}
},0)
//ExcelUtils.multiThreadingModify("D:\\数据\\小区\\5个城市小区列表0723(1).xlsx",{d->
// if(d['I'] && d['K']){
@@ -300,79 +303,79 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
//},0)
def index = 0
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\重庆反查数据_669(审核完毕).xlsx",{d->
if(d['A']){
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
def alias = []
if(d['H'] && d['H']!=dist.name){
alias << d['H']
if(d['H'].contains("·")){
alias << d['H'].replace("·", "")
}
if(dist.alias){
for(def aliasName : dist.alias){
if(!alias.contains(aliasName)){
alias << aliasName
}
}
}
}
def address = null
if(d['I']){
address = AddressUtil.formatAddress(dist.city, d['I'])
}
Long lng = null
Long lat = null
Long lat2 = null
Long lng2 = null
if(d['J']){
lng = d['J'].toString().toDouble() * 10000000000l;
lat = d['K'].toString().toDouble() * 10000000000l;
def json = BaiDuApiUtils.geoconv(d['J'].toString().toDouble(), d['K'].toString().toDouble(), 3)
if(json != null && json.containsKey("x")) {
lat2 = json.get("y") * 10000000000l
lng2 = json.get("x") * 10000000000l
}
}
def region = null
if(dist.region!=d['F']){
region = dist.region
}
def udata = new BasicDBObject()
if(region){
udata.put("region", region)
}
if(address){
udata.put("address", address)
}
if(lat2){
udata.put("lng", lng)
udata.put("lat", lat)
udata.put("lng2", lng2)
udata.put("lat2", lat2)
}
if(alias){
udata.put("alias", alias)
}
if(udata){
index++
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
println index+ " "+dist.city +" "+dist.name+" "+udata
}
}
}
},0)
//def index = 0
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\重庆反查数据_669(审核完毕).xlsx",{d->
// if(d['A']){
// def dist = CommonDatautils.findDistrictById(d['A'])
// if(dist){
//
// def alias = []
// if(d['H'] && d['H']!=dist.name){
// alias << d['H']
// if(d['H'].contains("·")){
// alias << d['H'].replace("·", "")
// }
// if(dist.alias){
// for(def aliasName : dist.alias){
// if(!alias.contains(aliasName)){
// alias << aliasName
// }
// }
// }
// }
//
// def address = null
// if(d['I']){
// address = AddressUtil.formatAddress(dist.city, d['I'])
// }
//
// Long lng = null
// Long lat = null
// Long lat2 = null
// Long lng2 = null
// if(d['J']){
// lng = d['J'].toString().toDouble() * 10000000000l;
// lat = d['K'].toString().toDouble() * 10000000000l;
// def json = BaiDuApiUtils.geoconv(d['J'].toString().toDouble(), d['K'].toString().toDouble(), 3)
// if(json != null && json.containsKey("x")) {
// lat2 = json.get("y") * 10000000000l
// lng2 = json.get("x") * 10000000000l
// }
// }
//
// def region = null
// if(dist.region!=d['F']){
// region = dist.region
// }
//
// def udata = new BasicDBObject()
// if(region){
// udata.put("region", region)
// }
//
// if(address){
// udata.put("address", address)
// }
// if(lat2){
// udata.put("lng", lng)
// udata.put("lat", lat)
// udata.put("lng2", lng2)
// udata.put("lat2", lat2)
// }
//
// if(alias){
// udata.put("alias", alias)
// }
// if(udata){
// index++
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
// println index+ " "+dist.city +" "+dist.name+" "+udata
// }
//
// }
// }
//
//},0)
@@ -440,7 +443,7 @@ ExcelUtils.multiThreadingModify("D:\\
def findDealCount(sqlInstance, def startDate, def endDate){
def sql = "select cityname,count(*) as count from bigdata_deal_gj_test where dealdate>=to_date('${startDate}','yyyy-mm-dd') and dealdate<to_date('${endDate}','yyyy-mm-dd') group by cityname";
def sql = "select cityname,count(*) as count from bigdata_deal_gj where dealdate>=to_date('${startDate}','yyyy-mm-dd') and dealdate<to_date('${endDate}','yyyy-mm-dd') group by cityname";
def count = 0
def map = [:]
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
@BatchOwner(Watcher.GaoJuan)
class ThirdPartyPriceBatch extends BaseBatch {
def sDate= 202110//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202108
def endDate = 202110
def sDate= 202111//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202109
def endDate = 202111
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
@@ -40,10 +40,10 @@ class TradeStatLog {
static main(args) {
def task = new TradeStatLog()
// task.run()//统计最近3个月抓取的成交数据量
// task.exportTrade()//导出未入库成交
// task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
// task.averageStat()
task.exportTrade()//导出未入库成交
// task.deleteDistrict()
// task.deleteDealData()
// task.deleteDealData2()
@@ -73,13 +73,13 @@ class TradeStatLog {
def run(){
def citys = CcbCommonUtils.citys;
def rets = []
rets << ["城市","8月总量","9月总量","10月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
rets << ["城市","9月总量","10月总量","11月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2021-08-01"
def date = "2021-09-01"
def endDate = "2021-10-01"
def newDate = "2021-11-01"
def startDate = "2021-09-01"
def date = "2021-10-01"
def endDate = "2021-11-01"
def newDate = "2021-12-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -129,7 +129,7 @@ class TradeStatLog {
}
ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\10月成交统计.xlsx", rets, "数据")
ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\11月成交统计.xlsx", rets, "数据")
println "表格生成完成"
}
@@ -22,10 +22,10 @@ class MatchMapData {
static main(args) {
def task = new MatchMapData()
// task.run()
task.run()
// task.run1()
// task.run2()
task.run3()
// task.run3()
// task.exportHouse()
}
@@ -129,10 +129,14 @@ class MatchMapData {
def run(){
def files = ["北京","广州","杭州","合肥","上海","深圳","石家庄","天津","武汉","西安","重庆"]
def files = ["合肥","上海","北京","广州","杭州","深圳","石家庄","天津","武汉","西安","重庆"]
for(def f : files){
def city = f
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\"+f+".xlsx",{d->
def lastYear = DateUtils.add(DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd"), -365)
def lastWeekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightYYYYMMDD(lastYear))
def districtPriceMap = CommonDatautils.queryAllDistrictPriceByWeekend(city, lastWeekend)
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\租售比\\"+f+".xlsx",{d->
if(d['E']){
def region = d['E']
def newRegion = RegionData.formatRegion(city, region)
@@ -148,12 +152,34 @@ class MatchMapData {
d['K']="映射别名"
d['L'] = dist.name_cat
}else{
dist = CommonDatautils.findDistrictByAlias(city, newRegion, d['F'],"district")
if(dist){
def rets = CommonDatautils.findDistrictByAlias(city, newRegion, d['F'],"district")
if(rets){
d['K']="别名"
}
if(rets){
dist = rets[0]
}
}
}
if(dist){
def nearDbo = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
def lastMonth = DateUtils.parseFromTightYYYYMMDD("2021-11-30")
def sqmetre = CommonDatautils.findSqmetreRent(dist.city, dist.region, dist.name, lastMonth)
if(nearDbo && districtPriceMap){
def priceObj = districtPriceMap.get(dist.region+"#"+dist.name)
if(priceObj){
def markup = (nearDbo.avg_price - priceObj.avg_price)*100/priceObj.avg_price
d['M'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, markup)
}
}
if(nearDbo && sqmetre){
def prc = nearDbo.avg_price/sqmetre.avg_price
d['N'] = prc.toInteger()
}
d['O'] = dist.estate_type2
}
}else{
d['M'] = "区域无法标准化"
}
@@ -32,7 +32,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
public static def batchsize = 1000
@@ -42,7 +42,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
@Override
public void process(Object args) {
def citys = ["苏州","北京","广州","杭州","合肥","深圳","石家庄","武汉","西安","重庆"]//CcbCommonUtils.citys;
def citys = CcbCommonUtils.citys;
def rows = []
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
// for (def city : citys) {
@@ -16,7 +16,7 @@ import java.sql.ResultSetMetaData
@BatchOwner(Watcher.GaoJuan)
class ExportDistrictRongHe extends BaseBatch{
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
runBatch(new ExportDistrictRongHe(), args);
@@ -31,20 +31,17 @@ class ExportDistrictRongHe extends BaseBatch{
// exportAllBigdataDealFJW()
for(def city : citys) {
def cityCode = cityCodeDataMap.get(city)
//// if(!codes.contains(cityCode)){
//// println city +" "+cityCode
//// }
exportBigdataCommunityGJRH(cityCode)
exportBigdataBuildingFJW(cityCode)
exportBigdataCommfeeGj(cityCode)
// exportBigdataCommunityGJRH(cityCode)
// exportBigdataBuildingFJW(cityCode)
// exportBigdataCommfeeGj(cityCode)
//
exportBigdataDealFJW(cityCode)
exportBigdataCommbuildstruGJ(cityCode)
exportBigdataCommhousetypeGJ(cityCode)
exportBigdataCommbuildyearGj(cityCode)
exportBigdataCommhouseuseGJ(cityCode)
exportBigdataPositionGJ(cityCode)
exportBigdataDistrictFJW(cityCode)
// exportBigdataCommbuildstruGJ(cityCode)
// exportBigdataCommhousetypeGJ(cityCode)
// exportBigdataCommbuildyearGj(cityCode)
// exportBigdataCommhouseuseGJ(cityCode)
// exportBigdataPositionGJ(cityCode)
// exportBigdataDistrictFJW(cityCode)
exportBigdataAliasFJW(cityCode)
//
// exportBigdataBuildprojFjw(cityCode)
@@ -66,6 +66,8 @@ class CommonDatautils {
public static final DBCollection POI_COLL = MongoUtils.getCollectionByName2("fangjia_base", "baidu_place_poi", mongoDbUrl)
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", mongoDbUrl)
public static final DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName2("fangjia", "sqmetre_rent", mongoDbUrl)
public static DSCollection getHouseDataStoreConnection(){
if(house_dataStoreConnection != null) {
return house_dataStoreConnection
@@ -3097,5 +3099,19 @@ class CommonDatautils {
def count = DEAL_PRICE_COLL.count(q);
return count;
}
static def findSqmetreRent(def city, def region, def name, def weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("district_name", name)
q.put("weekend", weekend)
def dbcur = SQMETRE_RENT_COLL.find(q)
def dbo = null
for(def d : dbcur){
dbo = d;
}
return dbo;
}
}
@@ -48,7 +48,7 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2021-10-01"
static def statDate = "2021-11-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
@@ -285,7 +285,7 @@ class YouDaDataParser {
def esfParser(){
def files = ["D:\\数据\\友达\\2021年11月\\二手房\\查询1.DBF"]
def files = ["D:\\数据\\友达\\2021年12月\\二手房\\查询1.DBF"]
def datas = []
for(def fileName : files){
def rows = readDBF(fileName);
@@ -38,8 +38,8 @@ class Esf58Crawl extends BaseBatch{
def static useDistKeys = []
static def totalCount = new AtomicLong()
//"福州":"fz|304","合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319","成都":"cd|102","上海":"sh|2", "北京":"bj|1","天津":"tj|18",
//"路헤":"cq|37","柯소鏤":"sjz|241","밤鹵":"gz|3","�柏":"sz|4",
def cityList = ["땜賻":"dg|413","뤘�":"fs|222","볼鹵":"hz|79","嶠볶":"wh|158",
//"路헤":"cq|37","柯소鏤":"sjz|241","밤鹵":"gz|3","�柏":"sz|4","땜賻":"dg|413","뤘�":"fs|222","볼鹵":"hz|79","嶠볶":"wh|158",
def cityList = [
"青岛":"qd|122","南京":"nj|172","嘉兴":"jx|497","大连":"dl|147","乌鲁木齐":"xj|984","太原":"ty|740","兰州":"lz|952","南昌":"nc|669",
"沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362","常州":"cz|463", "南通":"nt|394","吉林":"jl|700","中山":"zs|771","台州":"tz|403","东营":"dy|623","三亚":"sanya|2422",
"呼和浩特":"hu|811","济南":"jn|265","温州":"wz|330","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096",