diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy index 3cd74dc7f..a9ea036ef 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy @@ -97,7 +97,13 @@ class BigdataBuildingBatch extends BaseBatch{ }else{ ret << "" } - ret << CcbCodeUtils.parseBuildType(dist.build_type) + def arrs = CcbCodeUtils.parseBuildType(dist.build_type) + if(arrs){ + ret << arrs[0].trim() + }else{ + ret << '' + } + ret << CcbCodeUtils.getBuildStructureCode(dist.build_structure) if(obj.floors){ ret << obj.floors diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 7e09e80ec..6e470ce2c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -54,6 +54,7 @@ class BigdataCommunityBatch extends BaseBatch{ rows.addAll(rets) println city +" "+rets.size() + // FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false); } @@ -168,7 +169,7 @@ class BigdataCommunityBatch extends BaseBatch{ ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd") ret << dist.status ret << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss") - ret << "pa100" + ret << "pa101" ret << "" if(dist.address){ if(dist.address.toString().contains("弄")){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index dd9df53f4..03bc56f0a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.ccbevaluation +import com.fangjia.core.RegionData import com.fangjia.core.util.DataUtil import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner @@ -7,6 +8,7 @@ import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.ccb.CcbCodeUtils import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.gj.district.SeaweedDataMatchProcess import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils @@ -19,6 +21,7 @@ import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.atomic.AtomicLong import java.util.regex.Pattern @@ -29,8 +32,8 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataDealBatch extends BaseBatch{ - def curDate = 20210901//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + def curDate = 20211001//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"] public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; @@ -50,12 +53,15 @@ class BigdataDealBatch extends BaseBatch{ public void process(Object args) { //def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date()) def citys = CcbCommonUtils.citys; + +// for(def city : citys){ +// addDealPriceMatch(city); +// } println citys.unique().size() def rows = [] - def headers = ["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] + def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ - def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def ppeDists = [:] // if(["贵阳","佛山","杭州","沈阳","天津","中山","徐州","绍兴","南通","哈尔滨","惠州","长沙","青岛","长春","烟台","温州","太原","台州","石家庄","宁波","常州","大连","嘉兴","金华","昆明"].contains(city)){ @@ -76,12 +82,9 @@ class BigdataDealBatch extends BaseBatch{ rows.addAll(rets) println city +" 成交共:"+rets.size() - -// FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal_${city}_1.txt",headers, "utf-8", false); def cityDatas = [] cityDatas << headers cityDatas.addAll(rows) - // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal_${city}.xlsx", cityDatas, "数据") } def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" @@ -92,10 +95,7 @@ class BigdataDealBatch extends BaseBatch{ FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); println "txt 生成" -// def rets = [] -// rets << headers -// rets.addAll(rows) -// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal.xlsx", rets, "数据") + } def createData(def rets, cityCodeDatas, distCodeMap, source){ @@ -223,7 +223,6 @@ class BigdataDealBatch extends BaseBatch{ ret << ""//期数 String roomNum = DataUtil.match(trade.address, Pattern.compile("\\d+室")) if(roomNum && roomNum.contains("0")){ - ret << roomNum.substring(0, roomNum.lastIndexOf("0")) }else{ ret << trade.floor @@ -231,11 +230,16 @@ class BigdataDealBatch extends BaseBatch{ ret << "" ret << trade.height - ret << CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型 + def arrs = CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型 + def buildType = '' + if(arrs){ + buildType = arrs[0] + } + ret << buildType ret << CcbCodeUtils.getBuildStructureCode(obj.build_structure) ret << trade.area ret << "" - ret << getHouseRoom(trade.room, trade.hall, trade.toilet) + ret << getHouseRoom(buildType, trade.room, trade.hall) ret << trade.room ret << trade.hall ret << "" @@ -251,7 +255,7 @@ class BigdataDealBatch extends BaseBatch{ ret << "" ret << "" ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd") - ret << 2 + ret << "02" ret << "" ret << "" ret << "" @@ -336,11 +340,17 @@ class BigdataDealBatch extends BaseBatch{ ret << trade.floor ret << "" ret << trade.height - ret << CcbCodeUtils.parseBuildType(trade.build_type)//建筑类型 +// ret << CcbCodeUtils.parseBuildType(trade.build_type)//建筑类型 + def arrs = CcbCodeUtils.parseBuildType(trade.build_type)//建筑类型 + def buildType = '' + if(arrs){ + buildType = arrs[0] + } + ret << buildType ret << CcbCodeUtils.getBuildStructureCode(trade.build_structure) ret << trade.area ret << "" - ret << getHouseRoom(trade.room, trade.hall, trade.toilet) + ret << getHouseRoom(buildType, trade.room, trade.hall) ret << trade.room ret << trade.hall ret << "" @@ -356,7 +366,7 @@ class BigdataDealBatch extends BaseBatch{ ret << "" ret << "" ret << DateUtils.formatDateByPattern(trade.trade_date, "yyyy-MM-dd") - ret << 2 + ret << "02" ret << "" ret << "" ret << "" @@ -390,18 +400,38 @@ class BigdataDealBatch extends BaseBatch{ return ret; } - def getHouseRoom(def room, def hall, def toilet){ - def s = "" - if(room){ - s = s + room +"室" + def getHouseRoom(buildtype, def roomcnt, def livingroomcnt){ + def housetype = null + if(buildtype && ["0112","0110","0109"].contains(buildtype)){ + housetype = "07" + }else if(buildtype && ["0107","0106","0105"].contains(buildtype)){ + housetype = "16" + }else{ + if(roomcnt && roomcnt==4){ + if(livingroomcnt>1){ + housetype = "14" + }else{ + housetype = "11" + } + }else if(roomcnt && roomcnt==3){ + if(livingroomcnt>1){ + housetype = "13" + }else{ + housetype = "10" + } + }else if(roomcnt && roomcnt==2){ + if(livingroomcnt>1){ + housetype = "12" + }else{ + housetype = "09" + } + }else if(roomcnt && roomcnt==1){ + housetype = "08" + }else{ + housetype="99" + } } - if(hall){ - s = s + hall +"厅" - } - if(toilet){ - s = s + toilet +"卫" - } - return s; + return housetype; } def getCityData(def city, def statMap, def ppeDistsMap){ @@ -443,14 +473,19 @@ class BigdataDealBatch extends BaseBatch{ // endPrice = (int)(price*0.5 + price) // } // } - def trades = CcbCommonUtils.findTradesDetail(dist, startDate, endDate, startPrice, endPrice) - if(!trades && dist.address){ - def flag = dist.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$') - if(flag){ - trades = CcbCommonUtils.findTradesDetailByAddress(dist, startDate, endDate, startPrice, endPrice) - } - - } +// def trades = CcbCommonUtils.findTradesDetail(dist, startDate, endDate, startPrice, endPrice) +// if(!trades && dist.address){ +// def flag = dist.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$') +// if(flag){ +// trades = CcbCommonUtils.findTradesDetailByAddress(dist, startDate, endDate, startPrice, endPrice)//地址是小区名 +// +// if(!trades){ +// trades = CcbCommonUtils.findTradesDetailByAddressAlias(dist, startDate, endDate, startPrice, endPrice)//地址是小区名 +// } +// } +// +// } + def trades = CcbCommonUtils.findTradesDetailById(dist, startDate, endDate) if(trades){ // def unitPrice= ppeDistsMap.get(dist._id.toString()) // if(unitPrice){ @@ -540,4 +575,50 @@ class BigdataDealBatch extends BaseBatch{ return map } + + def addDealPriceMatch(city){ + def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) + def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) + def trades = CcbCommonUtils.findTradesByMonth(city, startDate, endDate) + + final CountDownLatch latch = new CountDownLatch(trades.size()) + def final AtomicInteger count_ = new AtomicInteger(0) + + for(def i=0;i=startPrice && d.avg_price<=endPrice){ + def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd") + if(!rets.contains(key)){ + rets << key + d.dist_id = dist._id.toString() + d.address = dist.address + d.master_name = dist.name + d.build_type = dist.build_type + d.build_structure = dist.build_structure + d.house_roperty = dist.house_roperty + d.property_life = dist.property_life + d.estate_type2 = dist.estate_type2 + lists << d + } + } + } + return lists + } + public static def findTrades(dist, startDate, endDate){ def q = new BasicDBObject() q.put("city", dist.city) @@ -234,6 +299,26 @@ class CcbCommonUtils { return set } + + public static def findAllDistrictMapByOracle(def cityid){ + def sql = "select CITYID,COMMID,COMMNAME,DISTID from bigdata_community_gj where cityId='${cityid}' and status=0 "; + def detailMap = [:] + try{ + sqlInstance.eachRow(sql,{r-> + def map = [:] + map.cityid = r.CITYID + map.commid = r.COMMID + map.commname = r.COMMNAME + map.distid = r.DISTID + detailMap.put(r.COMMID, map) + }); + }catch(Exception e){ + println "错误:"+e.getMessage() + } + + return detailMap + } + public static def findAllDistrictsByOracle(def cityid){ def sql = "select CITYID,COMMID,COMMNAME,COMMALIAS,DISTID,STREET,STREETNO,ADDRESS,LANDAREA,BUILDAREA,COMMTYPE,IFNEW,GREENINGRATE,PLOTRATIO,PARKINGCNT,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,WORKDATE,ZTS,LONGITUDE,LATITUDE from bigdata_community_gj where cityId='${cityid}' and status=0 "; def rows = [] @@ -281,7 +366,7 @@ class CcbCommonUtils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name","江南人家") +// q.put("name","长隆小区") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) @@ -374,4 +459,32 @@ class CcbCommonUtils { return map } + + + + public static def findTradesByMonth(city, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) + q.put("match_id", new BasicDBObject('$exists', false)) + def dbcursor = DEAL_PRICE_COLL.find(q) + def total = dbcursor.count() + def rets = new HashSet() + def lists = [] + println city +" "+total + for(def d : dbcursor){ + lists << ["_id":d._id,"city":d.city, "region":d.region, "name":d.district_name, "address":d.address, "match_id":d.match_id] + } + return lists + } + + public static def updateDealPrice(id, match_city, match_region, match_name, match_id){ + def udata = new BasicDBObject() + udata.put("match_city", match_city) + udata.put("match_region", match_region) + udata.put("match_name", match_name) + udata.put("match_id", match_id) + DEAL_PRICE_COLL.update(new BasicDBObject("_id",id), new BasicDBObject('$set',udata)) + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy index 445ee4331..62ace6306 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy @@ -22,10 +22,10 @@ class CcbFtpUtils { static def ftpUpload(outputPath_dist, month){ //上传到ftp sftpChannel = reConnect(); - if(month.toString().endsWith("28")){ - sftp.upload(ftpPath+ month+"/", outputPath_dist, sftpChannel); - }else{ + if(month.toString().contains("10/") || month.toString().endsWith("10")){ sftp.upload(ftpPath_rh+ month+"/", outputPath_dist, sftpChannel); + }else{ + sftp.upload(ftpPath+ month+"/", outputPath_dist, sftpChannel); } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy similarity index 55% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealPriceDataProcess.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index e6909a187..2c428f4aa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -1,26 +1,38 @@ package com.fangjia.script.ccbevaluation +import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import groovy.sql.Sql +import org.bson.types.ObjectId +/** + * 建行成交数据表优化 + */ class DealPriceDataProcess { def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + def sqlInstance2 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + static main(args) { def task = new DealPriceDataProcess() - task.process() +// task.process()//当成交pe值不好时运行,拉升成交价格及删除pe太大数据 +// task.addLianJiaPrice(); + task.deleteThirdPartyData() } void process() { - def yearMonth = 202110 + def yearMonth = 202111 def cityPeMap = getCityPPeIndex("D:\\数据\\建行\\房产估值\\成交对标标准.xlsx") def citys = CcbCommonUtils.citys; def calcutePpeMap = findCaluateCityPPE(yearMonth) cityPeMap.each{c,v-> - if(c=="呼和浩特"){ +// if(c=="呼和浩特"){ def retMap = calcutePpeMap.get(c) if(retMap && retMap.ppe_val){ def ppe = retMap.ppe_val @@ -33,7 +45,7 @@ class DealPriceDataProcess { processDealPrice(v, c, retMap.case_count, count.toInteger(), yearMonth) } } - } +// } } } @@ -96,7 +108,7 @@ class DealPriceDataProcess { def findLtePPeCount(city, year_month){ def sql = "select count(*) as count from avm_evaluation_item_gj where year_month=${year_month} and cityname='${city}' and data_type='成交' and pe<=15"; def count = 0 - sqlInstance.eachRow(sql,{r-> + sqlInstance2.eachRow(sql,{r-> count = r.count }) return count @@ -106,7 +118,7 @@ class DealPriceDataProcess { def findAllGtePPeDatas(city, yearMonth){ def sql = "select cityid,cityname,distid,commid,commname,unit_price,target_avg_price,pe,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate from avm_evaluation_item_gj where year_month=${yearMonth} and cityname='${city}' and data_type='成交' and pe>15 order by pe asc"; def rets = [] - sqlInstance.eachRow(sql,{r-> + sqlInstance2.eachRow(sql,{r-> def map = [:] map.cityname = r.cityname map.cityid=r.cityid @@ -127,7 +139,7 @@ class DealPriceDataProcess { def findCaluateCityPPE(def yearMonth){ def sql = "select * from avm_evaluation_gj where year_month=${yearMonth}"; def map = [:] - sqlInstance.eachRow(sql,{r-> + sqlInstance2.eachRow(sql,{r-> def retMap = [:] retMap.put("ppe_val", r.DEAL_PPE15_COVERAGE) retMap.put("case_count", r.DEAL_TEST_CASE_NUM) @@ -145,4 +157,102 @@ class DealPriceDataProcess { } return map } + + + def addLianJiaPrice(){ + def citys = ["杭州","广州","东莞"] + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def publishTime = '2021-10-01' + def month = 202110 + def rows = [] + for(def city : citys){ + def cityid = cityCodeDataMap.get(city) + def priceMap = findLianjiaAvgPriceCount(cityid, publishTime) + def commids = findExistsThirdPartyPrice(cityid, month) + + def distMap = CcbCommonUtils.findAllDistrictMapByOracle(cityCodeDataMap.get(city)) + + priceMap.each {commid, avg_price-> + if(!commids.contains(commid)){ + + def distDetailMap = distMap.get(commid) + if(distDetailMap){ + def ret = [] + ret << new ObjectId().toString() + ret << distDetailMap['cityid'] + ret << distDetailMap['distid'] + ret << commid + ret << distDetailMap['commname'] + ret << "04" + ret << "链家" + ret << month + ret << avg_price + ret << "" + ret << "" + ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + rows << ret + } + } + } + println city +" "+rows.size() + } + + def fileName = "/home/goojia/ccb/BIGDATA_THIRD_PARTY_PRICE_GJ_2.dat" + if(CommonDatautils.isWindow()){ + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\TBIGDATA_THIRD_PARTY_PRICE_GJ_2.dat" + } + def header = [] + FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false); + + } + + def findExistsThirdPartyPrice(cityid, yearMonth){ + def sql = "select commid from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityid}' and price_date=${yearMonth} and data_source='链家'" + def commids = [] + sqlInstance.eachRow(sql,{r-> + commids << r.commid + }); + return commids + } + + + def findLianjiaAvgPriceCount(cityid, publishTime){ + def sql = "select commid, avg(unitprice) as avg_price from bigdata_forsale_gj where cityid='${cityid}' and publishtime>=to_date('${publishTime}','yyyy-MM-dd') and datasource in ('1','4') group by commid"; + def map = [:] + sqlInstance.eachRow(sql,{r-> + map.put(r.commid, r.avg_price.toInteger()) + }); + return map + } + + + def deleteThirdPartyData(){ + def citys = ["东莞"] //,"杭州""广州", + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def publishTime = '2021-10-01' + def month = 202110 + def rows = [] + for(def city : citys) { + def rets = findPeGteThirdParty(city, 202111) + def total = 0 + for(def m : rets){ + def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}" + sqlInstance.executeUpdate(sql) + } + println city +" "+total + } + } + + def findPeGteThirdParty(cityName, yearmonth){ + def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>20" + def rets = [] + sqlInstance2.eachRow(sql,{r-> + def map = [:] + map.commid = r.commid + map.price = r.target_avg_price + rets << map + }); + return rets + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 5beecaf04..d6ba74db8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -14,6 +14,10 @@ class ExportOracleData { def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + def sqlInstance2 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + static main(args) { def task = new ExportOracleData() task.run() @@ -26,22 +30,24 @@ class ExportOracleData { // exportCommunityBuileType() // exportBigdataDistrictData() // exportBigdataCommunity() +// exportBigdataDeal("2021-10-01") // exportBigdataForsale("2021-10-19") -// exportGuidedPrice("2021-10-01","2021-11-01") + exportGuidedPrice() + exportCommunityGuidedPrice() + +// exportThirdPartyPrice()//废弃掉,老表 +// exportThirdPartyPrice2() -// exportThirdPartyPrice() + exportAvmEvaluationGJNB(202111) + exportAVMEvaluationItemGJNB(202111) + exportCommunityBasePrice(202111) - exportThirdPartyPrice2() -// exportBigdataDeal() -// exportCommunityGuidedPrice() -// exportAvmEvaluationGJNB() -// exportAVMEvaluationItemGJNB() -// exportCommunityBasePrice() -// exportCommunityBasePriceGj() -// exportCityPrice() -// exportRegionPrice() +// exportCityPrice(202111) +// exportRegionPrice(202111) + +// exportCommunityBasePriceGj(202111) // exportLastFile() } @@ -138,20 +144,26 @@ class ExportOracleData { } def exportCommunityBuileType() { - def sql = "select CITYID, COMMID, BUILDTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, WORKDATE from bigdata_commbuildtype_gj"; +// def citys = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"] +// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def allRows = [] - sqlInstance.eachRow(sql, { r -> - def data = [] - data << r.CITYID - data << r.COMMID - data << r.BUILDTYPE - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - data << r.WORKDATE - allRows << data - }); +// for(def city : citys){ +// def cityid = cityCodeDataMap.get(city) + def sql = "select CITYID, COMMID, BUILDTYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, WORKDATE from bigdata_commbuildtype_gj"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.BUILDTYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.WORKDATE + allRows << data + }); +// } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { @@ -203,8 +215,8 @@ class ExportOracleData { } - def exportGuidedPrice(def startDate, def endDate) { - def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE,AVG_PRICE, GUIDING_PERSON, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME ,SOURCECODE from guided_price_gj where dealdate>=to_date('${startDate}','yyyy-MM-dd') and dealdate def data = [] @@ -225,9 +237,9 @@ class ExportOracleData { allRows << data }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; - def fileName = "/ccb/MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.dat" + def fileName = "/ccb/MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_GUIDED_PRICE_GJ_ADD_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" } def headers = [] //["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"] @@ -240,23 +252,23 @@ class ExportOracleData { def exportCommunityGuidedPrice() { def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj"; def allRows = [] - sqlInstance.eachRow(sql, { r -> - def data = [] - data << r.ID - data << r.COMMID - data << r.COMMNAME - data << r.CITYID - data << r.CITYNAME - data << r.DISTID - data << r.DISTNAME - data << r.AVGPRICE - data << r.SOURCETYPE - data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME - data << r.SOURCECODE - allRows << data - }); +// sqlInstance.eachRow(sql, { r -> +// def data = [] +// data << r.ID +// data << r.COMMID +// data << r.COMMNAME +// data << r.CITYID +// data << r.CITYNAME +// data << r.DISTID +// data << r.DISTNAME +// data << r.AVGPRICE +// data << r.SOURCETYPE +// data << r.REMARK +// data << r.CREATETIME +// data << r.UPDATETIME +// data << r.SOURCECODE +// allRows << data +// }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { @@ -270,12 +282,12 @@ class ExportOracleData { println "COMMUNITY_GUIDE_PRICE_GJ 上传Ftp完成......" } - def exportBigdataDeal() { + def exportBigdataDeal(def dealDate) { def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, \n" + "HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, \n" + "HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, \n" + "PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, CJJE, WORKDATE\n" + - "from bigdata_deal_gj"; + "from bigdata_deal_gj where dealdate>=to_date('${dealDate}','yyyy-MM-dd')"; def allRows = [] sqlInstance.eachRow(sql, { r -> def data = [] @@ -405,12 +417,13 @@ class ExportOracleData { FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "THIRD_PARTY_PRICE_GJ 数据文件输出完成......" - //CcbFtpUtils.ftpUpload(fileName, curDate) + CcbFtpUtils.ftpUpload(fileName, curDate) + println "THIRD_PARTY_PRICE_GJ 上传数据文件完成......" } def exportThirdPartyPrice2() { - def sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from third_party_price"; + def sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ"; def allRows = [] sqlInstance.eachRow(sql, { r -> def data = [] @@ -429,17 +442,17 @@ class ExportOracleData { allRows << data }) def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; - def fileName = "/ccb/MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { // fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" - fileName = "D:\\MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + fileName = "D:\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" } def headers = [] //["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); - println "THIRD_PARTY_PRICE_GJ 数据文件输出完成......" - //CcbFtpUtils.ftpUpload(fileName, curDate) + println "BIGDATA_THIRD_PARTY_PRICE_GJ 数据文件输出完成......" + CcbFtpUtils.ftpUpload(fileName, curDate) } @@ -590,13 +603,13 @@ class ExportOracleData { } - def exportAvmEvaluationGJNB(){ + def exportAvmEvaluationGJNB(yearMonth){ def sql = "select ID,CITYID,CITYNAME, YEAR_MONTH, COMMUNITY_NUM, HAS_PRICE_COMMUNITY_NUM, PRICE_COVERAGE, ARTIFICIAL_CALC_NUM, ARTIFICIAL_CALC_COVERAGE, SAMPLE_CALC_NUM, SAMPLE_CALC_COVERAGE, \n" + "ANN_CALC_NUM, ANN_CALC_COVERAGE, TEST_NUM, TEST_COMM_NUM, TEST_COMM_COVERAGE, PPE15_COVERAGE, RATE_MEDIAN, COEFFICIENT,DEAL_TEST_CASE_NUM, DEAL_TEST_COMM_NUM, DEAL_TEST_COMM_COVERAGE, DEAL_PPE15_COVERAGE, \n" + "DEAL_COEFFICIENT, DEAL_RATE_MEDIAN, THIRD_TEST_CASE_NUM, THIRD_TEST_COMM_NUM, THIRD_TEST_COMM_COVERAGE, THIRD_PPE15_COVERAGE, THIRD_COEFFICIENT, THIRD_RATE_MEDIAN, \n" + - "to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from avm_evaluation_gj where YEAR_MONTH >= 202106"; + "to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from avm_evaluation_gj where YEAR_MONTH = ${yearMonth}"; def allRows = [] - sqlInstance.eachRow(sql,{r-> + sqlInstance2.eachRow(sql,{r-> def data = [] data << r.ID data << r.CITYID @@ -637,24 +650,28 @@ class ExportOracleData { def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" + fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" } def headers = []//["ID","CITYID","CITYNAME","YEAR_MONTH","COMMUNITY_NUM","HAS_PRICE_COMMUNITY_NUM","PRICE_COVERAGE","ARTIFICIAL_CALC_NUM","ARTIFICIAL_CALC_COVERAGE","SAMPLE_CALC_NUM","SAMPLE_CALC_COVERAGE", //"ANN_CALC_NUM","ANN_CALC_COVERAGE","TEST_NUM","TEST_COMM_NUM","TEST_COMM_COVERAGE","PPE15_COVERAGE","RATE_MEDIAN","COEFFICIENT","DEAL_TEST_CASE_NUM","DEAL_TEST_COMM_NUM","DEAL_TEST_COMM_COVERAGE","DEAL_PPE15_COVERAGE", //"DEAL_COEFFICIENT","DEAL_RATE_MEDIAN","THIRD_TEST_CASE_NUM","THIRD_TEST_COMM_NUM","THIRD_TEST_COMM_COVERAGE","THIRD_PPE15_COVERAGE","THIRD_COEFFICIENT","THIRD_RATE_MEDIAN","CREATETIME","UPDATETIME","SOURCECODE"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false); println "AVM_EVALUATION_GJNB 数据文件输出完成......" CcbFtpUtils.ftpUpload(fileName, curDate) + CcbFtpUtils.ftpUpload(fileName1, curDate) println "AVM_EVALUATION_GJNB 上传Ftp完成......" } - def exportAVMEvaluationItemGJNB(){ + def exportAVMEvaluationItemGJNB(yearMonth){ def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, CONFIDENCE,TARGET_AVG_PRICE, \n" + "to_char(DEALDATE,'yyyy-mm-dd hh24:mi:ss') as DEALDATE, UNIT_PRICE, BUILDAREA, FITMENT, HOUSEASPECT, FLOORNUM, FLOORCNT, DATA_TYPE, RATE, PE, \n" + - "to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME from avm_evaluation_item_gj where YEAR_MONTH >= 202106"; + "to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME from avm_evaluation_item_gj where YEAR_MONTH = ${yearMonth}"; def allRows = [] - sqlInstance.eachRow(sql,{r-> + sqlInstance2.eachRow(sql,{r-> def data = [] data << r.ID data << r.CITYID @@ -685,20 +702,24 @@ class ExportOracleData { def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" + fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat" } def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","CONFIDENCE","TARGET_AVG_PRICE","DEALDATE","UNIT_PRICE","BUILDAREA","FITMENT","HOUSEASPECT","FLOORNUM","FLOORCNT","DATA_TYPE","RATE","PE","CREATETIME"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false); println "AVM_EVALUATION_ITEM_GJNB 数据文件输出完成......" CcbFtpUtils.ftpUpload(fileName, curDate) + CcbFtpUtils.ftpUpload(fileName1, curDate) println "AVM_EVALUATION_ITEM_GJNB 上传Ftp完成......" } - def exportCommunityBasePrice(){ - def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj"; + def exportCommunityBasePrice(yearMonth){ + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth}"; def allRows = [] - sqlInstance.eachRow(sql,{r-> + sqlInstance2.eachRow(sql,{r-> def data = [] data << r.ID data << r.CITYID @@ -724,24 +745,63 @@ class ExportOracleData { def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" } def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false); println "COMMUNITY_BASE_PRICE_GJNB 数据文件输出完成......" CcbFtpUtils.ftpUpload(fileName, curDate) + CcbFtpUtils.ftpUpload(fileName1, curDate) println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......" } - def exportCommunityBasePriceGj(){ + def exportCommunityBasePriceGj(yearMonth){ + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth}"; + def allRows = [] + sqlInstance2.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.AVG_PRICE + data << r.CALC_TYPE + data << r.FAKE_RATE + data << r.DIFF_RATE + data << r.CONFI_RATE + data << r.CORR_RATE + data << r.CONFIDENCE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "COMMUNITY_BASE_PRICE_GJ 数据文件输出完成......" + CcbFtpUtils.ftpUpload(fileName, curDate) + println "COMMUNITY_BASE_PRICE_GJ 上传Ftp完成......" } - def exportCityPrice(){ - def sql = "select ID, CITYID, CITYNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from city_price_gj"; + def exportCityPrice(month){ + def sql = "select ID, CITYID, CITYNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from city_price_gj where year_month=${month}"; def allRows = [] - sqlInstance.eachRow(sql,{r-> + sqlInstance2.eachRow(sql,{r-> def data = [] data << r.ID data << r.CITYID @@ -756,9 +816,9 @@ class ExportOracleData { }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" - def fileName = "/ccb/MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat" + def fileName = "/ccb/MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat" } def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); @@ -767,10 +827,10 @@ class ExportOracleData { println "CITY_PRICE_GJ 上传Ftp完成......" } - def exportRegionPrice(){ - def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from DISTRICT_PRICE_GJ"; + def exportRegionPrice(month){ + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from DISTRICT_PRICE_GJ where year_month=${month}"; def allRows = [] - sqlInstance.eachRow(sql,{r-> + sqlInstance2.eachRow(sql,{r-> def data = [] data << r.ID data << r.CITYID @@ -787,9 +847,9 @@ class ExportOracleData { }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" - def fileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat" + def fileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat" } def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 7a069e74c..03893ad8b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -53,24 +53,24 @@ class GuiDingPriceBatch extends BaseBatch{ } //添加搜房的参考价 - def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]); +// def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]); // def datas = findFangDatas(districts) // def fangRets = createDatas(cityCodeDatas, distCodeMap, datas) // rows.addAll(fangRets) //添加保亭文昌的指导价 - def cityDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\文昌-保亭待调查小区价格 - 210902(1).xlsx") - def cityDataMap = [:] - for(def d : cityDatas){ - cityDataMap.put(d['A'], d['E']) - } - def personDatas = findPersonDatas(districts, cityDataMap); - def personRets = createDatas(cityCodeDatas, distCodeMap, personDatas) - rows.addAll(personRets) +// def cityDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\文昌-保亭待调查小区价格 - 210902(1).xlsx") +// def cityDataMap = [:] +// for(def d : cityDatas){ +// cityDataMap.put(d['A'], d['E']) +// } +// def personDatas = findPersonDatas(districts, cityDataMap); +// def personRets = createDatas(cityCodeDatas, distCodeMap, personDatas) +// rows.addAll(personRets) } - def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"] + def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"] FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\GUIDED_PRICE_GJ.dat", headers, "utf-8", false); // def rets = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy index 26db4158d..6f9584e40 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy @@ -36,7 +36,7 @@ class StagingDistrictBatch extends BaseBatch{ def results =[] - def headers = ["分期ID","小区ID","小区主期ID","备注","创建时间","更新时间","来源公司"] + def headers = []//["分期ID","小区ID","小区主期ID","备注","创建时间","更新时间","来源公司"] results << headers results.addAll(rows) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 97725f68b..6aabf6064 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -30,6 +30,9 @@ import java.util.regex.*; def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") +def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") @@ -56,14 +59,12 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business // println city +" "+count //} -//def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0927.xlsx" +//def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0927(1).xlsx" //def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\7月成交.xlsx",0,1) //def dealMap = [:] //for(def d : datas){ // dealMap.put(d['A'], d['B']) //} -// -// //ExcelUtils.multiThreadingModify(fileName,{d-> // if(d['C']){ // d['Q'] = dealMap.get(d['C']) @@ -97,7 +98,7 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business // if (city) { // def cityid = cityCodeDatas.get(city) // def total = findCount(cityid, sqlInstance) -// d['H'] = total +// d['P'] = total // } //},0) @@ -221,14 +222,14 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def //def fileName = "D:\\数据\\建行\\建行成交\\成交数据_1025 - 副本.xlsx" -//def dealMap = findDealCount(sqlInstance, "2021-06-01", "2021-07-01") -//def testMap = findCaluateCityPPE(sqlInstance, 202107) +//def dealMap = findDealCount(sqlInstance, "2021-10-01", "2021-11-01") +//def testMap = findCaluateCityPPE(sqlInstance94, 202111) //ExcelUtils.modify(fileName,{d-> // if(d['C']){ // def oldcount = dealMap.get(d['C']) -// d['F'] = oldcount +// d['N'] = oldcount // def casecount = testMap.get(d['C']) -// d['G'] = casecount +// d['O'] = casecount // println d // } //},0) @@ -243,7 +244,7 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def //},0) //0-正常 1-非小区,2-地块, 3-无效小区,7-自然村 9-高德比对王一期不存在删除 -//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\需要修正的一期数据.xlsx",{d-> +//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\27城一期数据.xlsx",{d-> // if(d['A']){ // def obj = CommonDatautils.findDistrictById(d['A']) // if(obj && obj.name.contains("一期")){ @@ -267,81 +268,111 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def //},0) -def index = 0 -ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\新增别名数据_3788.xlsx",{d-> +ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\分期数据删除_3144.xlsx",{d-> if(d['A']){ - def dist = CommonDatautils.findDistrictById(d['A']) - if(dist){ + def udata = new BasicDBObject() + udata.put("name_cat", 9) + udata.put("m_date",new Date()) - def alias = [] - if(d['H'] && d['H']!=dist.name){ - alias << d['H'] - if(d['H'].contains("·")){ - alias << d['H'].replace("·", "") - } - if(dist.alias){ - for(def aliasName : dist.alias){ - if(!alias.contains(aliasName)){ - alias << aliasName - } - } - } - } - - def address = null - if(d['I']){ - address = AddressUtil.formatAddress(dist.city, d['I']) - } - - Long lng = null - Long lat = null - Long lat2 = null - Long lng2 = null - if(d['J']){ - lng = d['J'].toString().toDouble() * 10000000000l; - lat = d['K'].toString().toDouble() * 10000000000l; - def json = BaiDuApiUtils.geoconv(d['J'].toString().toDouble(), d['K'].toString().toDouble(), 3) - if(json != null && json.containsKey("x")) { - lat2 = json.get("y") * 10000000000l - lng2 = json.get("x") * 10000000000l - } - } - - def region = null - if(dist.region!=d['F']){ - region = dist.region - } - - def udata = new BasicDBObject() - if(region){ - udata.put("region", region) - } - - if(address){ - udata.put("address", address) - } - if(lat2){ - udata.put("lng", lng) - udata.put("lat", lat) - udata.put("lng2", lng2) - udata.put("lat2", lat2) - } - - if(alias){ - udata.put("alias", alias) - } - if(udata){ - index++ - DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) - println index+ " "+dist.city +" "+dist.name+" "+udata - } - - } + def q = new BasicDBObject() + q.put("_id", new ObjectId(d['A'])) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) } - },0) +//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\价格过滤的小区.xlsx",{d-> +// if(d['C']){ +//// def udata = new BasicDBObject() +//// udata.put("name_cat", 0) +//// udata.put("m_date",new Date()) +//// +//// def q = new BasicDBObject() +//// q.put("_id", new ObjectId(d['C'])) +//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +// def count = findDistCount(d['C'], sqlInstance); +// if(count==0){ +// println d['C'] +// } +// } +//},0) + + +//def index = 0 +//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\新增别名数据_3788.xlsx",{d-> +// if(d['A']){ +// def dist = CommonDatautils.findDistrictById(d['A']) +// if(dist){ +// +// def alias = [] +// if(d['H'] && d['H']!=dist.name){ +// alias << d['H'] +// if(d['H'].contains("·")){ +// alias << d['H'].replace("·", "") +// } +// if(dist.alias){ +// for(def aliasName : dist.alias){ +// if(!alias.contains(aliasName)){ +// alias << aliasName +// } +// } +// } +// } +// +// def address = null +// if(d['I']){ +// address = AddressUtil.formatAddress(dist.city, d['I']) +// } +// +// Long lng = null +// Long lat = null +// Long lat2 = null +// Long lng2 = null +// if(d['J']){ +// lng = d['J'].toString().toDouble() * 10000000000l; +// lat = d['K'].toString().toDouble() * 10000000000l; +// def json = BaiDuApiUtils.geoconv(d['J'].toString().toDouble(), d['K'].toString().toDouble(), 3) +// if(json != null && json.containsKey("x")) { +// lat2 = json.get("y") * 10000000000l +// lng2 = json.get("x") * 10000000000l +// } +// } +// +// def region = null +// if(dist.region!=d['F']){ +// region = dist.region +// } +// +// def udata = new BasicDBObject() +// if(region){ +// udata.put("region", region) +// } +// +// if(address){ +// udata.put("address", address) +// } +// if(lat2){ +// udata.put("lng", lng) +// udata.put("lat", lat) +// udata.put("lng2", lng2) +// udata.put("lat2", lat2) +// } +// +// if(alias){ +// udata.put("alias", alias) +// } +// if(udata){ +// index++ +// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) +// println index+ " "+dist.city +" "+dist.name+" "+udata +// } +// +// } +// } +// +//},0) + + //def fileName = "C:\\Users\\gaojuan\\Desktop\\分期分区小区.xlsx" //def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() @@ -450,6 +481,18 @@ def findCount(cityid, sqlInstance){ return count } + + +def findDistCount(commid, sqlInstance){ + def sql = "select count(*) as count from bigdata_community_gj where commid='${commid}' and status=0"; + def count = 0 + + sqlInstance.eachRow(sql,{r-> + count = r.count + }); + return count +} + def findFiledCount(cityid, field, sqlInstance){ def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and status=0 and "+field+" is not null"; def count = 0 @@ -567,6 +610,10 @@ def findFiledCount(cityid, field, sqlInstance){ //} + + + + def isInArea(lng, lat, points){ List pointList = InsidePolygon.createPoint2(points); //Point point = new Point(lng, lat); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index ee52c745c..2c861766f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202109//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202107 - def endDate = 202109 + def sDate= 202110//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202108 + def endDate = 202110 def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") @@ -48,24 +48,22 @@ class ThirdPartyPriceBatch extends BaseBatch { def weekend = DateUtils.getFirstDayOfWeek(new Date()) for(def city : citys){ index++ -// def statMap = getDistrictStatByCity(city, weekend) + def statMap = getDistrictStatByCity(city, weekend) def cityCodeDatas = CcbCommonUtils.findCityCodeValue() -// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) -// -// def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"]) - def thirdPriceMap = findThirdPartyPrice(city, statMap) -// def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap) -// def rets = createCityDatas2(cityCodeDatas, distCodeMap, dists, thirdPriceMap) -// rows.addAll(rets) - def rets = createData3(cityCodeDatas.get(city)) - rows.addAll(rets) + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"]) + def thirdPriceMap = findThirdPartyPrice(city, statMap) + //def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap)//老第三方均价表(现不用) + def rets = createCityDatas1(cityCodeDatas, distCodeMap, dists, thirdPriceMap) +// def rets = createData3(cityCodeDatas.get(city))//新第三方均价表,要做2步,麻烦 + rows.addAll(rets) println index+" "+city+" "+rets.size() +" "+rows.size() } - def fileName = "/home/goojia/ccb/THIRD_PARTY_PRICE_GJ.dat" + def fileName = "/home/goojia/ccb/BIGDATA_THIRD_PARTY_PRICE_GJ.dat" if(CommonDatautils.isWindow()){ - fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\THIRD_PARTY_PRICE_GJ.dat" + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_THIRD_PARTY_PRICE_GJ.dat" } def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] @@ -183,6 +181,127 @@ class ThirdPartyPriceBatch extends BaseBatch { } + def createCityDatas1(cityCodeDatas, distCodeMap, dists, thirdPriceMap){ + def allRows = [] + for(def d : dists){ + + def third = thirdPriceMap.get(d.region+"##"+d.name) + if(third){ + def data = distCodeMap.get(d.region) + if(data){ + def dataMap = [:] + dataMap.COMMID = d._id.toString() + dataMap.COMMNAME = d.name + dataMap.CITYID = cityCodeDatas.get(d.city) + dataMap.CITYNAME = d.city + dataMap.DISTID = data.district_code + dataMap.DISTNAME = data.district + dataMap.YEAR_MONTH = sDate + def price_map = [:] + dataMap.price_map = price_map + if(third.zhugezhaofang_esf_price){ + price_map.zhuge_avg_price = third.zhugezhaofang_esf_price + } + if(third.anjuke_esf_price){ + price_map.anjuke_avg_price = third.anjuke_esf_price + } + if(third.soufang_esf_price){ + price_map.fang_avg_price = third.soufang_esf_price + } + if(third.lianjia_esf_price){ + price_map.lianjia_avg_price = third.lianjia_esf_price + } + if(third.cityhouse_esf_price){ + price_map.cityhouse_avg_price = third.cityhouse_esf_price + } + if(third.maitian_esf_price){ + price_map.maitian_avg_price = third.maitian_esf_price + } + if(third.beike_esf_price){ + price_map.ke_avg_price = third.beike_esf_price + } + if(third.get("365house_esf_price")){ + price_map.taofang_avg_price = third.get("365house_esf_price") + } + if(third.zhongyuan_esf_price){ + price_map.zhongyuan_avg_price = third.zhongyuan_esf_price + } + if(third.tuboshi_esf_price){ + price_map.tbs_avg_price = third.tuboshi_esf_price + } + if(third.qfang_esf_price){ + price_map.qfang_avg_price = third.qfang_esf_price + } + if(third.fangtan_esf_price){ + price_map.fangtan_avg_price = third.fangtan_esf_price + } + if(third.fangduoduo_esf_price){ + price_map.fangduoduo_avg_price = third.fangduoduo_esf_price + } + if(third.get("5i5j_esf_price")){ + price_map.wiwj_avg_price = third.get("5i5j_esf_price") + } + dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + allRows << dataMap + } + } + } + + def results = [] + for(def m : allRows){ + def priceMap = m.price_map + priceMap.each{k,v-> + def ret = [] + ret << new ObjectId().toString() + ret << m.CITYID + ret << m.DISTID + ret << m.COMMID + ret << m.COMMNAME + + def web = "" + if(k=="zhuge_avg_price"){ + web = "诸葛找房" + }else if(k=="anjuke_avg_price"){ + web = "安居客" + }else if(k=="fang_avg_price"){ + web = "房天下" + }else if(k=="lianjia_avg_price"){ + web = "链家" + }else if(k=="cityhouse_avg_price"){ + web = "禧泰数据" + }else if(k=="maitian_avg_price"){ + web = "麦田" + }else if(k=="ke_avg_price"){ + web = "贝壳" + }else if(k=="taofang_avg_price"){ + web = "淘房365" + }else if(k=="zhongyuan_avg_price"){ + web = "中原" + }else if(k=="tbs_avg_price"){ + web = "兔博士" + }else if(k=="qfang_avg_price"){ + web = "Q房" + }else if(k=="fangtan_avg_price"){ + web = "房探" + }else if(k=="fangduoduo_avg_price"){ + web = "房多多" + }else if(k=="wiwj_avg_price"){ + web = "我爱我家" + } + + ret << sourceMap.get(web) + ret << web + ret << m.YEAR_MONTH + ret << v + ret << "" + ret << "" + ret << m.updatetime + results << ret + } + } + return results + } + def createCityDatas2(cityCodeDatas, distCodeMap, dists, thirdPriceMap) { def rets = [] for (def d : dists) { @@ -208,7 +327,7 @@ class ThirdPartyPriceBatch extends BaseBatch { }else if(k=="lianjia_esf_price"){ web = "链家" }else if(k=="cityhouse_esf_price"){ - web = "城市房产" + web = "禧泰数据" }else if(k=="maitian_esf_price"){ web = "麦田" }else if(k=="beike_esf_price"){ @@ -343,7 +462,7 @@ class ThirdPartyPriceBatch extends BaseBatch { }else if(k=="lianjia_avg_price"){ web = "链家" }else if(k=="cityhouse_avg_price"){ - web = "城市房产" + web = "禧泰数据" }else if(k=="maitian_avg_price"){ web = "麦田" }else if(k=="ke_avg_price"){ @@ -383,6 +502,7 @@ class ThirdPartyPriceBatch extends BaseBatch { def findThirdPartyPrice(def city, def statMap){ def q = new BasicDBObject() q.put("city", city) + q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1)) def map = [:] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index f7fb84bc2..a565bc730 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -39,13 +39,14 @@ class TradeStatLog { static main(args) { def task = new TradeStatLog() - task.run()//统计最近3个月抓取的成交数据量 +// task.run()//统计最近3个月抓取的成交数据量 // task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) // task.averageStat() -// task.exportTrade() + task.exportTrade()//导出未入库成交 // task.deleteDistrict() // task.deleteDealData() +// task.deleteDealData2() // task.statTableCount() @@ -480,7 +481,7 @@ class TradeStatLog { rets << headers for(def city : citys){ - def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern("2021-09-01","yyyy-MM-dd"), DateUtils.parseFromTightByPattern("2021-10-01","yyyy-MM-dd")) + def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern("2021-10-01","yyyy-MM-dd"), DateUtils.parseFromTightByPattern("2021-11-01","yyyy-MM-dd")) // def trades = getYouDaData() //def statMap = getDistrictStatByCity(city, weekend) @@ -546,8 +547,10 @@ class TradeStatLog { reason = "数据重复" } } + if(!reason && t.area<20){ + reason = "面积过小" + } } - def tradeDate = DateUtils.formatDateByPattern(t.trade_date,"yyyy-MM-dd") //rets << [t._id.toString(), t.city, t.region, t.district_name, t.source, t.avg_price, t.area,t.address, tradeDate, reason, t.estate_type] @@ -556,8 +559,7 @@ class TradeStatLog { } println city +" "+rets.size() } - - ExcelUtils.write("d:\\数据\\成交未入库记录(9月).xlsx", rets,"数据") + ExcelUtils.write("d:\\数据\\成交未入库记录(10月).xlsx", rets,"数据") } def deleteDistrict(){ @@ -614,7 +616,7 @@ class TradeStatLog { } def findDealIds(city){ - def sql = "select * from bigdata_deal_gj where dealdate>=to_date('2021-09-01','yyyy-mm-dd') and dealdate @@ -638,7 +640,7 @@ class TradeStatLog { public static def findTradesDetail(city, startDate, endDate){ def q = new BasicDBObject() q.put("city", city) -// q.put("district_name","百合花园") +// q.put("district_name","长隆住宅小区") q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) def dbcursor = DEAL_PRICE_COLL.find(q) def rets = [] @@ -693,4 +695,13 @@ class TradeStatLog { }) } + + def deleteDealData2(){ + //sqlInstance.eachRow("select dealid from bigdata_deal_gj where dealdate>=to_date('2021-10-01','yyyy-MM-dd')",{r-> + sqlInstance.executeUpdate("delete from bigdata_deal_gj where dealdate>=to_date('2021-10-01','yyyy-MM-dd')") + // }) + + + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy index a36bdf5f7..87e8f37fa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy @@ -42,22 +42,22 @@ class BigdataCommunityGJRHBatch extends BaseBatch { @Override public void process(Object args) { - - def citys = ["大连","哈尔滨","呼和浩特","厦门"]//CcbCommonUtils.citys; + def citys = CcbCommonUtils.citys; def rows = [] def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() // for (def city : citys) { // def cityid = cityCodeDataMap.get(city) -// def sql ="select count(*) as count from BIGDATA_COMMUNITY_FJW where CITYID='${cityid}'" +// def sql ="select count(*) as count from BIGDATA_COMMUNITY_GJ where CITYID='${cityid}'" // sqlInstance.eachRow(sql, { r -> // println city +" "+r.count // }) // } -//// addBigdataDistrictFJW() - +// addBigdataDistrictFJW() + def index = 0 for (def city : citys) { + index++ def districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city)) // // for(int i=0;i<10;i++){ @@ -88,23 +88,24 @@ class BigdataCommunityGJRHBatch extends BaseBatch { dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2) } } - println city +" 小区加载完成 "+districts.size() + println index +" "+city +" 小区加载完成 "+districts.size() addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city)) - println city +" 1.小区表完成" + println index +" "+city +" 1.小区表完成" addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city)) - println city +" 2.小区建筑结构表" + println index +" "+city +" 2.小区建筑结构表" addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city)) - println city +" 3.小区房屋类型表" + println index +" "+city +" 3.小区房屋类型表" addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city)) - println city +" 4.小区建筑年代表" + println index +" "+city +" 4.小区建筑年代表" addBigdataCommfeeGj(districts, cityCodeDataMap.get(city)) - println city +" 5.小区费用表" + println index +" "+city +" 5.小区费用表" addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city)) - println city +" 6.小区房屋用途表" + println index +" "+city +" 6.小区房屋用途表" addBigdataPositionGJ(districts, cityCodeDataMap.get(city)) - println city + " 7.地理位置表" + println index +" "+ city + " 7.地理位置表" + +// addDealFieldInfo(distMap, cityCodeDataMap.get(city));//修改成交记录housestyle 一定要运行 -// addDealFieldInfo(distMap, cityCodeDataMap.get(city)); } } @@ -653,7 +654,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { } def updateDealHouseStyle(distMap, cityid){ - def sql = "select dealid,housestyle,buildtype,roomcnt,livingroomcnt,buildyear,address,street,streetno,commid from bigdata_deal_gj_fjw where cityid='${cityid}'"; + def sql = "select dealid,housestyle,buildtype,roomcnt,livingroomcnt,buildyear,address,street,streetno,commid from bigdata_deal_gj where cityid='${cityid}'"; try{ def index = 0 sqlInstance.eachRow(sql,{r-> @@ -695,21 +696,21 @@ class BigdataCommunityGJRHBatch extends BaseBatch { def buildyear = null if(dist){ // if(!r.address){ - if(r.street || r.streetno){ - address = r.street - if(r.streetno && r.streetno!="null"){ - address = address + r.streetno - } - }else if(r.street){ - address = dist.address + if(r.street || r.streetno){ + address = r.street + if(r.streetno && r.streetno!="null"){ + address = address + r.streetno } + }else if(r.street){ + address = dist.address + } // } if(!r.buildyear && dist.finish_building_date){ buildyear = CcbCodeUtils.getBuildYear(dist.finish_building_date); } } if(housetype || buildyear || address){ - def updateSql = "update bigdata_deal_gj_fjw set housestyle='${housetype}'" + def updateSql = "update bigdata_deal_gj set housestyle='${housetype}'" if(buildyear){ updateSql = updateSql + ",buildyear=${buildyear}" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/CcbRHDealFileCreateBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/CcbRHDealFileCreateBatch.groovy new file mode 100644 index 000000000..3d892d834 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/CcbRHDealFileCreateBatch.groovy @@ -0,0 +1,28 @@ +package com.fangjia.script.ccbevaluation.ronghe + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.ccbevaluation.CcbFtpUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils + +@BatchOwner(Watcher.GaoJuan) +class CcbRHDealFileCreateBatch extends BaseBatch{ + static main(args) { + runBatch(new CcbRHDealFileCreateBatch(), args); + } + + @Override + public void process(Object args) { + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" + } + FileUtils.batchCreateFile([], fileName, [], "utf-8", false); + CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}") + println "BIGDATA_COMMFEE_FJW 上传Ftp完成......" + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 0c5d156d6..40b3605c1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -24,24 +24,32 @@ class ExportDistrictRongHe extends BaseBatch{ @Override void process(Object args) { - def citys = ["大连","哈尔滨","呼和浩特","厦门"] + def citys = CcbCommonUtils.citys; def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + +// exportAllBigdataDealFJW() for(def city : citys) { def cityCode = cityCodeDataMap.get(city) - exportBigdataCommbuildstruGJ(cityCode) +//// if(!codes.contains(cityCode)){ +//// println city +" "+cityCode +//// } exportBigdataCommunityGJRH(cityCode) - exportBigdataCommhousetypeGJ(cityCode) - exportBigdataCommbuildyearGj(cityCode) - exportBigdataCommfeeGj(cityCode) - exportBigdataCommhouseuseGJ(cityCode) - exportBigdataPositionGJ(cityCode) - exportBigdataDistrictFJW(cityCode) - - exportBigdataDealFJW(cityCode) - exportBigdataBuildingFJW(cityCode) +// exportBigdataBuildingFJW(cityCode) +// exportBigdataCommfeeGj(cityCode) +// +// exportBigdataDealFJW(cityCode) +// exportBigdataCommbuildstruGJ(cityCode) +// exportBigdataCommhousetypeGJ(cityCode) +// exportBigdataCommbuildyearGj(cityCode) +// exportBigdataCommhouseuseGJ(cityCode) +// exportBigdataPositionGJ(cityCode) +// exportBigdataDistrictFJW(cityCode) +// exportBigdataAliasFJW(cityCode) +// +// exportBigdataBuildprojFjw(cityCode) +// exportBigdataCommlayoutpicFjw(cityCode) } - - exportEvaluateRate() +// exportEvaluateRate() } @@ -65,7 +73,7 @@ class ExportDistrictRongHe extends BaseBatch{ data << r.GREENINGRATE data << r.PLOTRATIO data << r.PARKINGCNT - data << r.INTRODUCTION + data << '' data << r.REMARK data << r.CREATETIME data << r.UPDATETIME @@ -167,7 +175,12 @@ class ExportDistrictRongHe extends BaseBatch{ data << r.COMPID data << r.FEE data << r.HOUSEUSE - data << r.BUILDTYPE + if(r.BUILDTYPE){ + def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } data << r.FEETYPE data << r.NOTE data << r.REMARK @@ -269,8 +282,39 @@ class ExportDistrictRongHe extends BaseBatch{ println "BIGDATA_DISTRICT_FJW 上传Ftp完成......" } + def exportBigdataAliasFJW(cityCode){ + def sql = "select city_id, ca_id,address_id,address_name, status, exp_dt, version, is_key_name, data_source, description,alias_grade,addr_type_cd,ifexamine,ifdisplay from ADDRESS_ALIAS_FJW where city_id='${cityCode}'"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.city_id + data << r.ca_id + data << r.address_id + data << r.address_name + data << r.status + data << r.exp_dt + data << r.version + data << r.is_key_name + data << r.data_source + data << r.description + data << r.alias_grade + data << r.addr_type_cd + data << r.ifexamine + data << r.ifdisplay + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}") + println "ADDRESS_ALIAS_FJW 上传Ftp完成......" + } + def exportBigdataDealFJW(cityCode){ - def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,WORKDATE from BIGDATA_DEAL_FJW where CITYID='${cityCode}'"; + def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where CITYID='${cityCode}' and dealdate>=to_date('2021-06-01','yyyy-MM-dd')"; def allRows = [] sqlInstance.eachRow(sql,{r-> def data = [] @@ -298,7 +342,12 @@ class ExportDistrictRongHe extends BaseBatch{ data << r.FLOORNUM data << r.FLOORTYPE data << r.FLOORCNT - data << r.BUILDTYPE + if(r.BUILDTYPE){ + def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } data << r.BUILDSTRU data << r.BUILDAREA data << r.LIVINGAREA @@ -334,8 +383,8 @@ class ExportDistrictRongHe extends BaseBatch{ data << r.AGENCYID data << r.AGENCYNAME data << r.REMARK - data << r.CREATETIME - data << r.UPDATETIME + data << r.CREATETIME1 + data << r.UPDATETIME1 data << r.SOURCECODE data << r.WORKDATE allRows << data @@ -351,6 +400,93 @@ class ExportDistrictRongHe extends BaseBatch{ } + + def exportAllBigdataDealFJW(){ + def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.DEALID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.STREET + data << r.STREETNO + data << r.COMMID + data << r.COMMNAME + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.UNITID + data << r.UNITNO + data << r.UNITNAME + data << r.HOUSEID + data << r.ADDRESS + data << r.BUILDYEAR + data << r.PROPTYPE + data << r.PROPYEARS + data << r.PERIODNUM + data << r.FLOORNUM + data << r.FLOORTYPE + data << r.FLOORCNT + if(r.BUILDTYPE){ + def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.BUILDSTRU + data << r.BUILDAREA + data << r.LIVINGAREA + data << r.HOUSESTYLE + data << r.ROOMCNT + data << r.LIVINGROOMCNT + data << r.KITCHENCNT + data << r.BATHROOMCNT + data << r.HOUSEUSE + data << r.HOUSESTRU + data << r.HOUSEASPECT + data << r.FITMENT + data << r.EVALUATENO + data << r.DEALUNITPRICE + data << r.DEALTOTALPRICE + data << r.DECLAREPRICE + data << r.EVALUATEPRICE + data << r.CONTRACTNO + data << r.DEALDATE + data << r.DEALTYPE + data << r.PAYTYPE + data << r.DEALPERIOD + data << r.IFELEVATOR + data << r.COMPAREAGARDEN + data << r.COMPAREATERRACE + data << r.COMPAREABALCONY + data << r.COMPAREABAYWINDOW + data << r.COMPAREAATTIC + data << r.COMPAREAGARAGE + data << r.COMPAREABASEMENT + data << r.COMPAREAOTHER + data << r.DATASOURCE + data << r.AGENCYID + data << r.AGENCYNAME + data << r.REMARK + data << r.CREATETIME1 + data << r.UPDATETIME2 + data << r.SOURCECODE + data << r.WORKDATE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/All/FJW_000000_All_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\All\\FJW_000000_All_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + println "BIGDATA_DEAL_FJW 导出完成......" + } + + def exportBigdataBuildingFJW(cityCode){ def sql = "select CITYID, BUILDID, BUILDNO, BUILDNAME, BUILDALIAS, DISTID, COMMID, BUILDYEAR, BUILDTYPE, BUILDSTRU, FLOORCNT, UNITCNT, HOUSECNT, IFELEVATOR, ELEVATORCNT, ELEVATORHOUSE, REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, VALID_FLAG from BIGDATA_BUILDING_FJW where CITYID='${cityCode}'"; def allRows = [] @@ -364,7 +500,12 @@ class ExportDistrictRongHe extends BaseBatch{ data << r.DISTID data << r.COMMID data << r.BUILDYEAR - data << r.BUILDTYPE + if(r.BUILDTYPE){ + def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } data << r.BUILDSTRU data << r.FLOORCNT data << r.UNITCNT @@ -414,7 +555,29 @@ class ExportDistrictRongHe extends BaseBatch{ } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload(fileName, curDate) - println "BIGDATA_DISTRICT_FJW 上传Ftp完成......" + println "EVALUATE_RATE_FJW 上传Ftp完成......" + } + + def exportBigdataBuildprojFjw(cityCode){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" + } + FileUtils.batchCreateFile([], fileName, [], "utf-8", false); + CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}") + println "BIGDATA_BUILDPROJ_FJW 上传Ftp完成......" + } + + def exportBigdataCommlayoutpicFjw(cityCode){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" + } + FileUtils.batchCreateFile([], fileName, [], "utf-8", false); + CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}") + println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......" } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/SSHHelper.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/SSHHelper.groovy new file mode 100644 index 000000000..af4571228 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/SSHHelper.groovy @@ -0,0 +1,112 @@ +package com.fangjia.script.ccbevaluation.ronghe + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; + +import com.jcraft.jsch.ChannelExec; +import com.jcraft.jsch.JSch; +import com.jcraft.jsch.Session + +import com.jcraft.jsch.*; + + +class SSHHelper { + + private Session session = null; + private Channel channel = null; + + private String sftpHost = "192.168.0.45"; + private int sftpPort = 17495; + private String sftpUserName = "ccb"; + private String sftpPassword = "ccb!@#"; + private int timeout = 30000; + /** + * 获取连接 + * @return + */ + private ChannelExec getChannelExec() { + try { + if (channel != null && channel.isConnected()) { + return (ChannelExec) channel; + } + JSch jSch = new JSch(); + if (session == null || !session.isConnected()) { + session = jSch.getSession(sftpUserName, sftpHost, sftpPort); + session.setPassword(sftpPassword); + Properties config = new Properties(); + config.put("StrictHostKeyChecking", "no"); + session.setConfig(config); + session.setTimeout(timeout); + session.connect(); + } + channel = session.openChannel("exec"); + } catch (Exception e) { + if (session != null) { + session.disconnect(); + session = null; + } + channel = null; + } + return channel == null ? null : (ChannelExec) channel; + } + + /** + * 关闭连接 + */ + private void closeChannel() { + try { + if (channel != null) { + channel.disconnect(); + channel = null; + } + if (session != null) { + session.disconnect(); + session = null; + } + } catch (Exception e) { + System.out.println(e); + } + } + + /** + * 执行服务器端命令 + */ + public boolean executeCommand(String command) { + boolean flag = false; + ChannelExec channelExec = getChannelExec(); + if (channelExec == null) { + return false; + } + try { + channelExec.setInputStream(null); + channelExec.setErrStream(System.err); + channelExec.setCommand(command); + + InputStream ins = channelExec.getInputStream(); + channelExec.connect(); + + BufferedReader reader = new BufferedReader(new InputStreamReader(ins, "utf-8")); + String line = null; + while ((line = reader.readLine()) != null) { + System.out.println(line); + } + + reader.close(); + closeChannel(); + + flag = true; + } catch (Exception e) { + System.out.println(e); + flag = false; + } + return flag; + } + + public static void main(String[] args) { + SSHHelper jschCommand = new SSHHelper(); + System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/package.sh 20211210")); + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 821cc9765..87c6ba7ed 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -705,6 +705,7 @@ class CommonDatautils { q.put("visible",0) q.put("cat", cat); q.put("city", city); +// q.put("name","长隆小区") DBObject keys = new BasicDBObject(); if(fields){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsHouseMatchDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsHouseMatchDistrict.groovy new file mode 100644 index 000000000..050e3aa9f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsHouseMatchDistrict.groovy @@ -0,0 +1,120 @@ +package com.fangjia.script.gj.es + +import com.fangjia.core.RegionData +import com.fangjia.core.util.HttpRequest +import com.fangjia.httpclient.UserAgents +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.JacksonUtil +import net.sf.json.JSONObject +import org.apache.commons.lang.StringUtils +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger +import java.util.zip.GZIPInputStream + +class EsHouseMatchDistrict { + + def targetUrl = "http://192.168.0.206:9200/seawater-2021/_doc" + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + + static main(args) { + def task = new EsHouseMatchDistrict() + task.houseRun() + } + + def houseRun(){ + def citys = ["苏州","杭州"]//CommonUtils.findAllCity(); + for(def city : citys){ + def endDate = DateUtils.truncateDay(new Date()) + def startDate = DateUtils.parseFromTightByPattern("2020-12-01","yyyy-MM-dd")//DateUtils.truncateDay(new Date()-1) + def houses = findHouseCounts(city, startDate, endDate); + match(city, houses); + } + } + + + def match(city, houses){ + final CountDownLatch latch = new CountDownLatch(houses.size()) + final AtomicInteger count = new AtomicInteger(0) + final AtomicInteger successCount = new AtomicInteger(0) + for(def i=0;i < houses.size();i++){ + def t = houses[i] + fixedThreadPool.execute( + new Runnable() { + void run() { + count.addAndGet(1) + try{ + if(!t.match_seaweed_id){ + def rMap = SeaweedDataMatchProcess.doMatch(t.city, t.region, t.district_name, null); + if(rMap){ + successCount.addAndGet(1) + updateHouseMatchDistrictId(t.id, rMap.id) + } + println count.get()+" "+t.id+" "+ rMap + } + }catch(Exception e){ + e.printStackTrace() + }finally { + latch.countDown() + } + + } + } + ); + } + latch.await() + if(houses){ + def pre = DataHandleUtils.getDoubleRound(1, successCount.get()*100.0/houses.size()); + println city +" "+houses.size()+" "+new Date().toString()+" "+pre+"%" + } + + + } + + def findHouseCounts(def city, def startDate, def endDate){ + QueryBuilder date_range = QueryBuilders.rangeQuery("m_date").from(startDate).to(endDate).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", "sell")).must(date_range) +// BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); +// for(def s : ["sell", "rent"]){ +// inBuilder.should(QueryBuilders.matchQuery("cat", s)); +// } +// queryBuilder.must(inBuilder) + + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + return houses + } + + + def updateHouseMatchDistrictId(def id, def match_id){ + def header = [:] + header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("developer:5e8edd851d2fdfbd7415232c67367cc3".getBytes("UTF-8"))}".toString() + String seawaterUrl = "http://192.168.0.206:9200/seawater-2021/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false" + def body = """ + { + "script": { + "source": "ctx._source.match_seaweed_id='${match_id}'", + "lang": "painless" + }, + "query": { + "term": { + "_id": "${id}" + } + } + } + """ + HttpRequest request = new HttpRequest() + request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8") + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index 89b54feea..6dedf2c7b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -752,7 +752,7 @@ class DataStandardBatch extends BaseBatch{ if(regionMap){ regionMap.each{k,v-> def arrays = k.split("###") - if(arrays[0]){ + if(arrays[0] && arrays.size()==2){ def py2 = com.fangjia.core.util.DataUtil.hanyuQuanpin(arrays[0], " ") if(py1 == py2){ districts << [city:city, region:region, name:arrays[0],address:arrays[1]] @@ -782,7 +782,6 @@ class DataStandardBatch extends BaseBatch{ def findAllNoMatchData(){ DBObject qdata = new BasicDBObject(); qdata.put("s_date", SHORT_DATE) - qdata.put("city", "福州") qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false)); def ret = [] try{ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index d7e1021de..83adfa245 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -37,41 +37,40 @@ class Esf58Crawl extends BaseBatch{ def static final distMap = [:] def static useDistKeys = [] static def totalCount = new AtomicLong() -//"福州":"fz|304","合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319", -// "安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","深圳":"sz|4","北京":"bj|1","上海":"sh|2","天津":"tj|18","重庆":"cq|37","广州":"gz|3","东莞":"dg|413","佛山":"fs|222", - def cityList = ["杭州":"hz|79", "成都":"cd|102","石家庄":"sjz|241","青岛":"qd|122","南京":"nj|172","嘉兴":"jx|497","大连":"dl|147","乌鲁木齐":"xj|984","太原":"ty|740", - "西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362","常州":"cz|463", + //"福州":"fz|304","合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319","成都":"cd|102", + def cityList = [ "上海":"sh|2","北京":"bj|1","天津":"tj|18","重庆":"cq|37","石家庄":"sjz|241","广州":"gz|3","深圳":"sz|4","东莞":"dg|413","佛山":"fs|222","杭州":"hz|79", + "青岛":"qd|122","南京":"nj|172","嘉兴":"jx|497","大连":"dl|147","乌鲁木齐":"xj|984","太原":"ty|740","兰州":"lz|952","南昌":"nc|669", + "沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362","常州":"cz|463", "南通":"nt|394","吉林":"jl|700","中山":"zs|771","台州":"tz|403","东营":"dy|623","三亚":"sanya|2422", + "呼和浩特":"hu|811","济南":"jn|265","温州":"wz|330","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096", + "临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385","盐城":"yancheng|613","烟台":"yt|228","信阳":"xy|8694","南充":"nanchong|2378","洛阳":"luoyang|556","南阳":"ny|592","厦门":"xm|606", + "宜昌":"yc|858","张家口":"zjk|3328","昆明":"km|541","柳州":"liuzhou|7133","绵阳":"mianyang|1057", "泉州":"qz|291","济宁":"jining|450","贵阳":"gy|2015","连云港":"lyg|2049","邯郸":"hd|572", "淮北":"huaibei|9357","滁州":"chuzhou|10266", "马鞍山":"mas|2039","铜陵":"tongling|10285","宣城":"xuancheng|5633","亳州":"bozhou|2329","黄山":"huangshan|2323", "池州":"chizhou|10260","巢湖":"ch|10229","和县":"hexian|10892","霍邱":"hq|11226","桐城":"tongcheng|11296", "宁国":"ningguo|5645","天长":"tianchang|10273","东至":"dongzhi|10262","无为":"wuweixian|10232", - "厦门":"xm|606","泉州":"qz|291","莆田":"pt|2429","漳州":"zhangzhou|710", - "宁德":"nd|7951","三明":"sm|2048","南平":"np|10291","龙岩":"ly|6752","武夷山":"wuyishan|10761", + "莆田":"pt|2429","漳州":"zhangzhou|710","宁德":"nd|7951","三明":"sm|2048","南平":"np|10291","龙岩":"ly|6752","武夷山":"wuyishan|10761", "石狮":"shishi|296","晋江":"jinjiangshi|297","南安":"nananshi|293","龙海":"longhai|713","上杭":"shanghangxian|6757", "福安":"fuanshi|7969","福鼎":"fudingshi|7970","安溪":"anxixian|7100","永春":"yongchunxian|7101","永安":"yongan|2133", - "漳浦":"zhangpu|717", "中山":"zs|771", - "珠海":"zh|910","惠州":"huizhou|722","江门":"jm|629","汕头":"st|783","湛江":"zhanjiang|791", + "漳浦":"zhangpu|717", "珠海":"zh|910","惠州":"huizhou|722","江门":"jm|629","汕头":"st|783","湛江":"zhanjiang|791", "肇庆":"zq|901","茂名":"mm|679","揭阳":"jy|927","梅州":"mz|9389","清远":"qingyuan|7303", "阳江":"yj|2284","韶关":"sg|2192","河源":"heyuan|10467","云浮":"yf|10485","汕尾":"sw|9449", "潮州":"chaozhou|10461","台山":"taishan|11263","阳春":"yangchun|8566","顺德":"sd|8716","惠东":"huidong|725", "博罗":"boluo|726","海丰":"haifengxian|9444","开平":"kaipingshi|634","陆丰":"lufengshi|9456", - "南宁":"nn|845","柳州":"liuzhou|7133","桂林":"gl|1039","玉林":"yulin|2337","梧州":"wuzhou|2046", + "南宁":"nn|845","桂林":"gl|1039","玉林":"yulin|2337","梧州":"wuzhou|2046", "北海":"bh|10536","贵港":"gg|6770","钦州":"qinzhou|2335","百色":"baise|10513","河池":"hc|2340", "来宾":"lb|10552","贺州":"hezhou|10549","防城港":"fcg|10539","崇左":"chongzuo|10524","桂平":"guipingqu|6774", "北流":"beiliushi|9168","博白":"bobaixian|9173","岑溪":"cenxi|2119", - "贵阳":"gy|2015","遵义":"zunyi|7620","黔东南":"qdn|9363","黔南":"qn|10492","六盘水":"lps|10506", + "遵义":"zunyi|7620","黔东南":"qdn|9363","黔南":"qn|10492","六盘水":"lps|10506", "毕节":"bijie|10564","铜仁":"tr|10417","安顺":"anshun|7468","黔西南":"qxn|10434","仁怀":"renhuaishi|7628", - "清镇":"qingzhen|12703", - "兰州":"lz|952","天水":"tianshui|8601","白银":"by|10304","庆阳":"qingyang|10475","平凉":"pl|7154", + "清镇":"qingzhen|12703","天水":"tianshui|8601","白银":"by|10304","庆阳":"qingyang|10475","平凉":"pl|7154", "酒泉":"jq|10387","张掖":"zhangye|10454","武威":"wuwei|10448","定西":"dx|10322","金昌":"jinchang|7428", "陇南":"ln|10415","临夏":"linxia|7112","嘉峪关":"jyg|10362","甘南":"gn|10343","敦煌":"dunhuang|10390", - "海口":"haikou|2053","三亚":"sanya|2422","五指山":"wzs|9952","三沙":"sansha|13722","琼海":"qh|10136", + "海口":"haikou|2053","五指山":"wzs|9952","三沙":"sansha|13722","琼海":"qh|10136", "文昌":"wenchang|9984","万宁":"wanning|10022","屯昌":"tunchang|10044","琼中":"qiongzhong|10064","陵水":"lingshui|10184", "东方":"df|10250","定安":"da|10303","澄迈":"cm|10331","保亭":"baoting|10367","白沙":"baish|10380", - "儋州":"danzhou|10394", - "郑州":"zz|342","洛阳":"luoyang|556","新乡":"xx|1016","南阳":"ny|592","许昌":"xc|977", - "平顶山":"pds|1005","安阳":"ay|1096","焦作":"jiaozuo|3266","商丘":"sq|1029","开封":"kaifeng|2342", - "濮阳":"puyang|2346","周口":"zk|933","信阳":"xy|8694","驻马店":"zmd|1067","漯河":"luohe|2347", + "儋州":"danzhou|10394", "郑州":"zz|342","新乡":"xx|1016","许昌":"xc|977", + "平顶山":"pds|1005","焦作":"jiaozuo|3266","开封":"kaifeng|2342", + "濮阳":"puyang|2346","周口":"zk|933","驻马店":"zmd|1067","漯河":"luohe|2347", "三门峡":"smx|9317","鹤壁":"hb|2344","济源":"jiyuan|9918","明港":"mg|8541","鄢陵":"yanling|9123", "禹州":"yuzhou|979","长葛":"changge|9344","灵宝":"lingbaoshi|9307","杞县":"qixianqu|7389","汝州":"ruzhou|1010", "项城":"xiangchengshi|935","偃师":"yanshiqu|7121","长垣":"changyuan|5936","滑县":"huaxian|5405","林州":"linzhou|1101", @@ -83,8 +82,7 @@ class Esf58Crawl extends BaseBatch{ "哈尔滨":"hrb|202","大庆":"dq|375","齐齐哈尔":"qqhr|5853","牡丹江":"mdj|3489","绥化":"suihua|6718", "佳木斯":"jms|6776","鸡西":"jixi|7289","双鸭山":"sys|9837","鹤岗":"hegang|9061","黑河":"heihe|9862", "伊春":"yich|9773","七台河":"qth|9848","大兴安岭":"dxal|9878","安达":"shanda|6720","肇东":"shzhaodong|6721", - "肇州":"zhaozhou|382", - "武汉":"wh|158","宜昌":"yc|858","襄阳":"xf|891","荆州":"jingzhou|3479","十堰":"shiyan|2032", + "肇州":"zhaozhou|382", "武汉":"wh|158","襄阳":"xf|891","荆州":"jingzhou|3479","十堰":"shiyan|2032", "黄石":"hshi|1734","孝感":"xiaogan|3434","黄冈":"hg|2299","恩施":"es|2302","荆门":"jingmen|2296", "咸宁":"xianning|9617","鄂州":"ez|9709","随州":"suizhou|9656","潜江":"qianjiang|9669","天门":"tm|9517", "仙桃":"xiantao|9736","神农架":"snj|9605","宜都":"yidou|864","汉川":"hanchuan|3439","枣阳":"zaoyang|896", @@ -99,16 +97,13 @@ class Esf58Crawl extends BaseBatch{ "祁东":"qidongxian|5690","衡东":"hengdong|5693","冷水江":"lengshuijiangshi|9470","涟源":"lianyuanshi|9471","双峰":"shuangfengxian|9473", "邵阳县":"shaoyangxian|6955","邵东":"shaodongxian|6954","沅江":"yuanjiangs|10201","南县":"nanxian|10202","祁阳":"qiyang|8532", "湘阴":"xiangyin|828","华容":"huarong|830","慈利":"cilixian|6791","攸县":"zzyouxian|1095", - "保定":"bd|424","唐山":"ts|276","廊坊":"lf|772","邯郸":"hd|572", - "秦皇岛":"qhd|1078","沧州":"cangzhou|652","邢台":"xt|751","衡水":"hs|993","张家口":"zjk|3328", + "保定":"bd|424","廊坊":"lf|772","秦皇岛":"qhd|1078","沧州":"cangzhou|652","邢台":"xt|751","衡水":"hs|993", "承德":"chengde|6760","定州":"dingzhou|8398","馆陶":"gt|8706","张北":"zhangbei|11201","赵县":"zx|9048", "正定":"zd|3198","迁安市":"qianan|284","任丘":"renqiu|656","三河":"sanhe|776","武安":"wuan|577", "雄安新区":"xionganxinqu|111234","燕郊":"lfyanjiao|12804","涿州":"zhuozhou|428","河间":"hejian|658","黄骅":"huanghua|657", "沧县":"cangxian|659","磁县":"cixian|591","涉县":"shexian|14059","霸州":"bazhou|775","香河":"xianghe|5395", "固安":"lfguan|12803","遵化市":"zunhua|283","迁西":"qianxixian|7061","玉田":"yutianxian|7060","滦南":"luannanxian|7066", - "沙河":"shaheshi|755", - "徐州":"xz|471", - "南通":"nt|394","扬州":"yz|637","盐城":"yancheng|613","淮安":"ha|968","连云港":"lyg|2049", + "沙河":"shaheshi|755", "徐州":"xz|471", "扬州":"yz|637","淮安":"ha|968", "泰州":"taizhou|693","宿迁":"suqian|2350","镇江":"zj|645","沭阳":"shuyang|5772","大丰":"dafeng|11279", "如皋":"rugao|397","启东":"qidong|400","溧阳":"liyang|469","海门":"haimen|399","东海":"donghai|2147", "扬中":"yangzhong|649","兴化":"xinghuashi|699","新沂":"xinyishi|478","泰兴":"taixing|696","如东":"rudong|402", @@ -116,29 +111,27 @@ class Esf58Crawl extends BaseBatch{ "东台":"dongtai|615","丹阳":"danyang|648","宝应县":"baoyingx|14451","灌南":"guannan|2150","灌云":"guanyun|2148", "姜堰":"jiangyan|697","金坛":"jintan|468","昆山":"szkunshan|16","泗洪":"sihong|5958","泗阳":"siyang|5959", "句容":"jurong|650","射阳":"sheyang|621","阜宁":"funingxian|620","响水":"xiangshui|619","盱眙":"xuyi|976", - "金湖":"jinhu|975","江阴":"jiangyins|34984", - "南昌":"nc|669","赣州":"ganzhou|2363","九江":"jj|2247","宜春":"yichun|5709","吉安":"ja|2364", + "金湖":"jinhu|975","江阴":"jiangyins|34984", "赣州":"ganzhou|2363","宜春":"yichun|5709","吉安":"ja|2364", "上饶":"sr|10120","萍乡":"px|2248","抚州":"fuzhou|10134","景德镇":"jdz|2360","新余":"xinyu|10115", "鹰潭":"yingtan|3209","永新":"yxx|11077","乐平":"lepingshi|9048","进贤":"jinxian|677","分宜":"fenyi|10118", "丰城":"fengchengshi|5711","樟树":"zhangshu|5713","高安":"gaoan|5712","余江":"yujiang|3210","南城":"nanchengx|10137", "浮梁":"fuliangxian|9071", - "长春":"cc|319","吉林":"jl|700","四平":"sp|10171","延边":"yanbian|3184","松原":"songyuan|2315", + "长春":"cc|319","四平":"sp|10171","延边":"yanbian|3184","松原":"songyuan|2315", "白城":"bc|5918","通化":"th|10159","白山":"baishan|10179","辽源":"liaoyuan|2501","公主岭":"gongzhuling|10171", "梅河口":"meihekou|10162","扶余":"fuyuxian|9085","长岭":"changlingxian|9084","桦甸":"huadian|706","磐石":"panshi|708", "梨树县":"lishu|10176", - "沈阳":"sy|188","鞍山":"as|523","锦州":"jinzhou|2354","抚顺":"fushun|5722", + "鞍山":"as|523","锦州":"jinzhou|2354","抚顺":"fushun|5722", "营口":"yk|5898","盘锦":"pj|2041","朝阳":"cy|10106","丹东":"dandong|3445","辽阳":"liaoyang|2038", "本溪":"benxi|5845","葫芦岛":"hld|10088","铁岭":"tl|6729","阜新":"fx|10097","庄河":"pld|3306", "瓦房店":"wfd|3279","灯塔":"dengta|2071","凤城":"fengcheng|3450","北票":"beipiao|10109","开原":"kaiyuan|6733", "银川":"yinchuan|2054","吴忠":"wuzhong|9962","石嘴山":"szs|9971","中卫":"zw|9951","固原":"guyuan|2421", - "呼和浩特":"hu|811","包头":"bt|801","赤峰":"chifeng|6700","鄂尔多斯":"erds|2037","通辽":"tongliao|10015", + "包头":"bt|801","赤峰":"chifeng|6700","鄂尔多斯":"erds|2037","通辽":"tongliao|10015", "呼伦贝尔":"hlbe|10039","巴彦淖尔市":"bycem|10070","乌兰察布":"wlcb|9993","锡林郭勒":"xl|2408","兴安盟":"xam|9976", "乌海":"wuhai|2404","阿拉善盟":"alsm|10083","海拉尔":"hlr|2043", "西宁":"xn|2052","海西":"hx|9902","海北":"haibei|9917","果洛":"guoluo|9936","海东":"haidong|9909", "黄南":"huangnan|9896","玉树":"ys|9888","海南":"hainan|10574","格尔木":"geermushi|9904", - "济南":"jn|265","烟台":"yt|228","临沂":"linyi|505", - "淄博":"zb|385","济宁":"jining|450","泰安":"ta|686","聊城":"lc|882","威海":"weihai|518", - "枣庄":"zaozhuang|961","德州":"dz|728","日照":"rizhao|3177","东营":"dy|623","菏泽":"heze|5632", + "泰安":"ta|686","聊城":"lc|882", + "枣庄":"zaozhuang|961","德州":"dz|728","日照":"rizhao|3177","菏泽":"heze|5632", "滨州":"bz|944","莱芜":"lw|2292","章丘":"zhangqiu|8680","垦利":"kl|11313","诸城":"zc|9146", "寿光":"shouguang|369","龙口":"longkou|233","曹县":"caoxian|5638","单县":"shanxian|5636","肥城":"feicheng|690", "高密":"gaomi|371","广饶":"guangrao|627","桓台":"huantaixian|7335","莒县":"juxian|3180","莱州":"laizhou|235", @@ -151,14 +144,14 @@ class Esf58Crawl extends BaseBatch{ "汶上":"wenshang|460","梁山":"liangshanx|462","利津":"lijin|628","沂南":"yinanxian|7301","栖霞":"qixia|238", "宁阳":"ningyang|691","东平":"dongping|692","昌邑":"changyishi|372","安丘":"anqiu|370","昌乐":"changle|373", "临朐":"linqu|374","鄄城":"juancheng|5635", - "临汾":"linfen|5669","大同":"dt|6964","运城":"yuncheng|5653","晋中":"jz|8854", + "运城":"yuncheng|5653","晋中":"jz|8854", "长治":"changzhi|6921","晋城":"jincheng|3350","阳泉":"yq|8760","吕梁":"lvliang|3222","忻州":"xinzhou|3453", "朔州":"shuozhou|9871","临猗":"linyixian|9193","清徐":"qingxu|10908","柳林":"liulin|3225","高平":"gaoping|3354", "泽州":"zezhou|3353","襄垣":"xiangyuanxian|6928","孝义":"xiaoyi|3227", - "咸阳":"xianyang|7453","宝鸡":"baoji|2044","渭南":"wn|5733","汉中":"hanzhong|3163", + "宝鸡":"baoji|2044","渭南":"wn|5733","汉中":"hanzhong|3163", "榆林":"yl|5942","延安":"yanan|8973","安康":"ankang|3157","商洛":"sl|9854","铜川":"tc|9832", "神木":"shenmu|5944","韩城":"hancheng|5735","府谷":"fugu|5945","靖边":"jingbian|5947","定边":"dingbian|5948", - "绵阳":"mianyang|1057","德阳":"deyang|2373","南充":"nanchong|2378","宜宾":"yb|2380", + "德阳":"deyang|2373","宜宾":"yb|2380", "自贡":"zg|6745","乐山":"ls|3237","泸州":"luzhou|2372","达州":"dazhou|9799","内江":"scnj|5928", "遂宁":"suining|9688","攀枝花":"panzhihua|2371","眉山":"ms|9704","广安":"ga|2381","资阳":"zy|6803", "凉山":"liangshan|9717","广元":"guangyuan|9749","雅安":"ya|9687","巴中":"bazhong|9811","阿坝":"ab|9817", @@ -170,12 +163,12 @@ class Esf58Crawl extends BaseBatch{ "图木舒克":"tmsk|9559","库尔勒":"kel|7168","阿勒泰":"alt|18837","塔城":"tac|18845", "拉萨":"lasa|2055","日喀则":"rkz|9615","山南":"sn|9576","林芝":"linzhi|9646","昌都":"changdu|9648", "那曲":"nq|9618","阿里":"al|9678","日土":"rituxian|9682","改则":"gaizexian|9684", - "昆明":"km|541","曲靖":"qj|2389","大理":"dali|2398","红河":"honghe|2394","玉溪":"yx|2040", + "曲靖":"qj|2389","大理":"dali|2398","红河":"honghe|2394","玉溪":"yx|2040", "丽江":"lj|2392","文山":"ws|2395","楚雄":"cx|2393","西双版纳":"bn|2397","昭通":"zt|9409", "德宏":"dh|9437","普洱":"pe|9444","保山":"bs|2390","临沧":"lincang|9422","迪庆":"diqing|9432", "怒江":"nujiang|9462","弥勒":"milexian|8892","安宁":"anningshi|547","宣威":"xuanwushi|7533", - "宁波":"nb|135","温州":"wz|330","金华":"jh|531", - "台州":"tz|403","绍兴":"sx|355","湖州":"huzhou|831","丽水":"lishui|7921","衢州":"quzhou|6793", + "宁波":"nb|135","金华":"jh|531", + "绍兴":"sx|355","湖州":"huzhou|831","丽水":"lishui|7921","衢州":"quzhou|6793", "舟山":"zhoushan|8481","乐清":"yueqingcity|13950","瑞安":"ruiancity|13951","义乌":"yiwu|12291","余姚":"yuyao|5333", "诸暨":"zhuji|357","象山":"xiangshanxian|6738","温岭":"wenling|408","桐乡":"tongxiang|502","慈溪":"cixi|5334", "长兴":"changxing|834","嘉善":"jiashanx|14357","海宁":"haining|500","德清":"deqing|835","东阳":"dongyang|536", diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index e870f26ab..94f0e3a4c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -17,7 +17,7 @@ class SynchDistrictPrice { def districtDetailColl = CrawalDBUtil.getDistrictDetailColl(); // def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房", // "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"] - def sourceMap = ["fang.com":"搜房"] + def sourceMap = ["cityhouse.cn":"城市房产"] static main(args) { def task = new SynchDistrictPrice() task.run() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index 80d802c97..d28d0da9d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -12,10 +12,10 @@ class TestCrawlTotalCount { task.run(); } def run(){ - def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"] + def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] for(def source :sources){ - def count1 = findTotalCount(source, 202109); - def count2 = findTotalCount(source, 202110); + def count1 = findTotalCount(source, 202110); + def count2 = findTotalCount(source, 202111); println source +" "+count1 +" "+count2 } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy index ddc282cb5..7bf88e160 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy @@ -4,8 +4,10 @@ import com.fangjia.core.util.DataUtil import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher +import com.fangjia.script.ccbevaluation.CcbCommonUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.OtherPriceUtils +import com.fangjia.script.gj.ccb.CcbCodeUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject @@ -42,7 +44,7 @@ class ThirdDataToGatherBatch extends BaseBatch{ @Override void process(Object args) { println SHORT_DATE - def citys = CommonDatautils.findAllCity(); + def citys = CcbCommonUtils.citys;//CommonDatautils.findAllCity(); def weekend = DateUtils.getFirstDayOfWeek(new Date()); //def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightYYYYMMDD("2021-07-15")); def cityIndex = 0; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ZhugefangCrawl4.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ZhugefangCrawl4.groovy index a57c51cc7..fee1de15f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ZhugefangCrawl4.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ZhugefangCrawl4.groovy @@ -278,7 +278,7 @@ class ZhugefangCrawl4 extends BaseBatch{ int goCount = 0; while(goCount < 10){ try{ - def rs = doPost("http://192.168.0.19:32222/",null, params, true, "utf-8") + def rs = doPost("http://192.168.0.216:32222/",null, params, true, "utf-8") if (rs) { JSONObject jsonObject = JSONObject.fromObject(rs); if(jsonObject.status_code==200){