Merge branch 'script_gj' into 'master'

update

See merge request !254
This commit is contained in:
2022-01-22 08:53:10 +08:00
10 changed files with 485 additions and 156 deletions
@@ -13,8 +13,7 @@ import org.bson.types.ObjectId
*/ */
class DealPriceDataProcess { class DealPriceDataProcess {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def sqlInstance2 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", def sqlInstance2 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
@@ -31,16 +30,25 @@ class DealPriceDataProcess {
} }
def delDealByPe(){ def delDealByPe(){
def yearMonth = 202112 def yearMonth = 202201
def thirdMonth = 202111 def thirdMonth = 202112
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\估值模型测试202112_1223 - 副本.xlsx",0,1)
// def rets = findAllDealByPe(yearMonth, 70)
// for(def r : rets){
// deleteDealPrice(r)
// }
def list = findAllThirdByPe(yearMonth, 70)
for(def r : list){
deletThirdPrice(r, thirdMonth)
}
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\估值模型测试202201_1221 - 副本.xlsx",0,1)
for(def d : datas){ for(def d : datas){
if(d['G']){ // if(d['G']){
def rets = findDealByPe(d['A'], yearMonth, d['G']) // def rets = findDealByPe(d['A'], yearMonth, d['G'])
for(def r : rets){ // for(def r : rets){
deleteDealPrice(r) // deleteDealPrice(r)
} // }
} // }
// if(d['H']){ // if(d['H']){
// def rets = findCityByPpeRange(d['A'], yearMonth, 15, 20) // def rets = findCityByPpeRange(d['A'], yearMonth, 15, 20)
@@ -161,6 +169,26 @@ class DealPriceDataProcess {
return count return count
} }
def findAllDealByPe(yearMonth, peVal){
def sql = "select cityid,cityname,distid,commid,commname,unit_price,target_avg_price,pe,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate from avm_evaluation_item_gj where year_month=${yearMonth} and data_type='成交' and pe>${peVal}";
def rets = []
sqlInstance236.eachRow(sql,{r->
def map = [:]
map.cityname = r.cityname
map.cityid=r.cityid
map.distid = r.distid
map.commid = r.commid
map.commname = r.commname
map.deal_price = r.target_avg_price
map.unit_price = r.unit_price
map.buildarea = r.buildarea
map.ppe = r.pe
map.dealdate = r.dealdate
rets << map
})
return rets
}
def findDealByPe(city, yearMonth, peVal){ def findDealByPe(city, yearMonth, peVal){
def sql = "select cityid,cityname,distid,commid,commname,unit_price,target_avg_price,pe,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate from avm_evaluation_item_gj where year_month=${yearMonth} and cityname='${city}' and data_type='成交' and pe>${peVal}"; def sql = "select cityid,cityname,distid,commid,commname,unit_price,target_avg_price,pe,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate from avm_evaluation_item_gj where year_month=${yearMonth} and cityname='${city}' and data_type='成交' and pe>${peVal}";
def rets = [] def rets = []
@@ -181,6 +209,20 @@ class DealPriceDataProcess {
return rets return rets
} }
def findAllThirdByPe(yearMonth, peVal){
def sql = "select year_month, commid, target_avg_price from avm_evaluation_item_gj where year_month=${yearMonth} and data_type='第三方' and pe>${peVal}";
def rets = []
sqlInstance236.eachRow(sql,{r->
def map = [:]
map.year_month = r.year_month
map.commid = r.commid
map.target_avg_price = r.target_avg_price
rets << map
})
return rets
}
def findThirdByPe(city, yearMonth, peVal){ def findThirdByPe(city, yearMonth, peVal){
def sql = "select year_month, commid, target_avg_price from avm_evaluation_item_gj where year_month=${yearMonth} and cityname='${city}' and data_type='第三方' and pe>${peVal}"; def sql = "select year_month, commid, target_avg_price from avm_evaluation_item_gj where year_month=${yearMonth} and cityname='${city}' and data_type='第三方' and pe>${peVal}";
def rets = [] def rets = []
@@ -24,21 +24,21 @@ class ExportOracleData {
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
//不会变 // //不会变
exportEvaluationRateData() // exportEvaluationRateData()
exportBigdataPoiData() // exportBigdataPoiData()
exportBigdataDistrictData() // exportBigdataDistrictData()
//
exportGuidedPrice()//随人工改变 // exportGuidedPrice()//随人工改变
exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策 // exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策
//
//处理小区情况 // //处理小区情况
exportBigdataCommunity() // exportBigdataCommunity()
exportCommunityStaging() // exportCommunityStaging()
exportCommunityBuileType() // exportCommunityBuileType()
exportThirdPartyPrice2(lastMonth) exportThirdPartyPrice2(lastMonth)
exportBigdataForsale("2021-12-19") // exportBigdataForsale("2021-12-19")
exportBigdataDeal("2021-12-01") exportBigdataDeal("2021-12-01")
@@ -879,7 +879,7 @@ class ExportOracleData {
// //
// for(def city : citys) { // for(def city : citys) {
// def cityCode = cityCodeDataMap.get(city) // def cityCode = cityCodeDataMap.get(city)
def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj"; def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month}";
def allRows = [] def allRows = []
sqlInstance.eachRow(sql, { r -> sqlInstance.eachRow(sql, { r ->
def data = [] def data = []
@@ -927,7 +927,7 @@ class ExportOracleData {
allRows << data allRows << data
}); });
println allRows.size() println allRows.size()
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
// } // }
println "community_p_describe_gj 数据文件输出完成......" println "community_p_describe_gj 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, curDate) CcbFtpUtils.ftpUpload(fileName, curDate)
@@ -948,7 +948,7 @@ class ExportOracleData {
for(def city : citys){ for(def city : citys){
def cityCode = cityCodeDataMap.get(city) def cityCode = cityCodeDataMap.get(city)
def sql = "select ID, CITYID, COMMUNITY_ID, NETLISTING_ID, PRICE_DATE, to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE from comm_case_detail_gj where cityid='${cityCode}'";//price_date=${month} and def sql = "select ID, CITYID, COMMUNITY_ID, NETLISTING_ID, PRICE_DATE, to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE from comm_case_detail_gj where cityid='${cityCode}' and price_date=${month}";
def allRows = [] def allRows = []
sqlInstance.eachRow(sql,{r-> sqlInstance.eachRow(sql,{r->
def data = [] def data = []
@@ -224,18 +224,18 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
} }
//def fileName = "D:\\数据\\建行\\房产估值\\成交数据_1222 - 副本.xlsx" def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx"
//def dealMap = findDealCount(sqlInstance236, "2021-11-01", "2021-12-01") def dealMap = findDealCount(sqlInstance236, "2021-12-01", "2022-01-01")
//def testMap = findCaluateCityPPE(sqlInstance236, 202112) def testMap = findCaluateCityPPE(sqlInstance236, 202201)
//ExcelUtils.modify(fileName,{d-> ExcelUtils.modify(fileName,{d->
// if(d['C']){ if(d['C']){
// def oldcount = dealMap.get(d['C']) def oldcount = dealMap.get(d['C'])
// d['R'] = oldcount d['T'] = oldcount
// def casecount = testMap.get(d['C']) def casecount = testMap.get(d['C'])
// d['S'] = casecount d['U'] = casecount
// println d println d
// } }
//},0) },0)
//ExcelUtils.multiThreadingModify("D:\\数据\\小区\\5个城市小区列表0723(1).xlsx",{d-> //ExcelUtils.multiThreadingModify("D:\\数据\\小区\\5个城市小区列表0723(1).xlsx",{d->
// if(d['I'] && d['K']){ // if(d['I'] && d['K']){
@@ -39,8 +39,8 @@ class TradeStatLog {
static main(args) { static main(args) {
def task = new TradeStatLog() def task = new TradeStatLog()
task.run()//统计最近3个月抓取的成交数据量 // task.run()//统计最近3个月抓取的成交数据量
// task.exportTrade()//导出未入库成交 task.exportTrade()//导出未入库成交
// task.thirdPartyStat() // task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR) // println Calendar.getInstance().get(Calendar.YEAR)
// task.averageStat() // task.averageStat()
@@ -129,7 +129,7 @@ class TradeStatLog {
} }
ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\12月成交统计.xlsx", rets, "数据") ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\12月成交统计.xlsx", rets, "数据")
println "表格生成完成" println "表格生成完成"
} }
@@ -479,16 +479,18 @@ class TradeStatLog {
def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"] def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"]
def rets = [] def rets = []
rets << headers rets << headers
def startDate = "2021-12-01"
def endDate = "2022-01-01"
for(def city : citys){ for(def city : citys){
def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern("2021-10-01","yyyy-MM-dd"), DateUtils.parseFromTightByPattern("2021-11-01","yyyy-MM-dd")) def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
// def trades = getYouDaData() // def trades = getYouDaData()
//def statMap = getDistrictStatByCity(city, weekend) //def statMap = getDistrictStatByCity(city, weekend)
def nameMap = getAllDistrict(city) def nameMap = getAllDistrict(city)
def rMap = findDealIds(city); def rMap = findDealIds(city, startDate, endDate);
def ids = rMap.ids def ids = rMap.ids
def details = rMap.detail def details = rMap.detail
@@ -559,7 +561,7 @@ class TradeStatLog {
} }
println city +" "+rets.size() println city +" "+rets.size()
} }
ExcelUtils.write("d:\\数据\\成交未入库记录(10月).xlsx", rets,"数据") ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(12月).xlsx", rets,"数据")
} }
def deleteDistrict(){ def deleteDistrict(){
@@ -615,8 +617,8 @@ class TradeStatLog {
return trades return trades
} }
def findDealIds(city){ def findDealIds(city, startDate, endDate){
def sql = "select * from bigdata_deal_gj where dealdate>=to_date('2021-10-01','yyyy-mm-dd') and dealdate<to_date('2021-11-01','yyyy-mm-dd') and cityname='${city}'";//and dealdate<to_date('2021-08-01','yyyy-mm-dd') def sql = "select * from bigdata_deal_gj where dealdate>=to_date('${startDate}','yyyy-mm-dd') and dealdate<to_date('${endDate}','yyyy-mm-dd') and cityname='${city}'";//and dealdate<to_date('2021-08-01','yyyy-mm-dd')
def ids = [] def ids = []
def detailInfos = [] def detailInfos = []
sqlInstance.eachRow(sql,{r-> sqlInstance.eachRow(sql,{r->
@@ -11,10 +11,14 @@ import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import groovy.sql.Sql
DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43") DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\人工核实_8511.xlsx", { d -> //ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\人工核实_8511.xlsx", { d ->
// def newRegion = RegionData.formatRegion(d['A'], d['B']) // def newRegion = RegionData.formatRegion(d['A'], d['B'])
// if(newRegion && !d['K']) { // if(newRegion && !d['K']) {
@@ -332,7 +336,7 @@ def city = "
//def rets = [] //def rets = []
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\国信达名字加入映射别名.xlsx", { d -> //ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\国信达名字加入映射别名.xlsx", { d ->
// if(d['A']){ // if(d['A'] && d['E'].toString().contains("鑫康花园北区")){
// def newRegion = RegionData.formatRegion(d['A'], d['B']) // def newRegion = RegionData.formatRegion(d['A'], d['B'])
// if(newRegion){ // if(newRegion){
// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") // def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
@@ -357,7 +361,7 @@ def city = "
//ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\新增映射别名.xlsx", rets,"数据") //ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\新增映射别名.xlsx", rets,"数据")
def rets = [] //def rets = []
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\查询别名表,将国信达数据增加到映射别名表里.xlsx", { d -> //ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\查询别名表,将国信达数据增加到映射别名表里.xlsx", { d ->
// if(d['A']){ // if(d['A']){
// def newRegion = RegionData.formatRegion(d['A'], d['B']) // def newRegion = RegionData.formatRegion(d['A'], d['B'])
@@ -388,15 +392,64 @@ def rets = []
//ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\新增映射别名_2.xlsx", rets,"数据") //ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\新增映射别名_2.xlsx", rets,"数据")
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\修改状态的数据.xlsx", { d -> //ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\修改状态的数据.xlsx", { d ->
if(d['A']) { // if(d['A']) {
def newRegion = RegionData.formatRegion(d['A'], d['B']) // def newRegion = RegionData.formatRegion(d['A'], d['B'])
if (newRegion) { // if (newRegion) {
def dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['C'], "district") // def dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['C'], "district")
if (dist) { // if (dist) {
rets << [dist._id.toString(), d['C']] // rets << [dist._id.toString(), d['C']]
} // }
// }
// }
//})
//ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\映射别名存在——修改状态.xlsx", rets,"数据")
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\修改后结果数据.xlsx",{d->
// if(d['A']){
// def dist = CommonDatautils.findDistrictById(d['B'])
// if(dist){
// d['E'] = dist.city
// d['F'] = dist.region
// d['G'] = dist.name
// d['H'] = dist.estate_type2
// d['I'] = dist.name_cat
// d['J'] = dist.status
// }
// }
//})
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\修改后结果数据.xlsx",{d->
// if(d['B']) {
// def sql = "select count(*) as count from bigdata_community_gj where commid='${d['B']}'";
// def count = 0;
// sqlInstance.eachRow(sql, { r ->
// count = r.count
// })
// if(count>=1){
// d['J'] = "oracle存在"
// }
// }
//})
def index = 0
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\漏映射别名.xlsx", { d ->
if (d['B']) {
def dist = CommonDatautils.findDistrictFieldById(d['B'],["city","region","name","mapping_alias"])
if(dist){
def alias = dist.mapping_alias
if(!alias){
alias = []
}
if(alias && !alias.contains(d['C'])){
index++
println index
alias << d['C']
def udata = new BasicDBObject()
udata.put("mapping_alias",alias)
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
}
} }
} }
}) })
ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\映射别名存在——修改状态.xlsx", rets,"数据")
@@ -0,0 +1,78 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.bson.types.ObjectId
class MatchDealPriceRegion {
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
static main(args) {
def task = new MatchDealPriceRegion();
task.run()
}
def run(){
def index = 0
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\成交\\部分未匹配成交.xlsx",{d->
def newRegion = null
if(d['C']){
newRegion = RegionData.formatRegion(d['B'], d['C'])
}
if(!newRegion){
index++
newRegion = findRegionByThirdPrice(d['B'], d['D']);
if(!newRegion || (newRegion && newRegion==d['C'])){
def rets = CommonDatautils.findDistrictsByName(d['B'], d['D'],"district")
if(rets && rets.size()==1){
newRegion = rets[0].region
}
}
if(newRegion && newRegion!=d['C']){
def query = new BasicDBObject()
query.put("_id", new ObjectId(d['A']))
def data = new BasicDBObject()
data.put("region", newRegion)
data.put("m_date", new Date())
println index+" "+d['A']+" "+d['C']+" "+newRegion
DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true)
}
}
})
}
def findRegionByThirdPrice(def city, def name){
def q = new BasicDBObject()
q.put("city", city)
q.put("name", name)
def dbcursor = collection.find(q).sort(new BasicDBObject("s_date",-1)).limit(10)
def map = [:]
for(def d : dbcursor){
def count = map.get(d.region)
if(!count){
count = 0
}
map.put(d.region, count+1)
}
if(map){
if(map.size()>1){
map = map.sort(){ a, b ->
b.value.compareTo(a.value)
}
}
def key = map.keySet().getAt(0)
return key
}
return null
}
}
@@ -27,15 +27,18 @@ class ExportDistrictRongHe extends BaseBatch{
def citys = CcbCommonUtils.citys; def citys = CcbCommonUtils.citys;
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
//检查ok文件是否上传
// //检查ok文件是否上传
// exportAllBigdataDealFJW() // exportAllBigdataDealFJW()
for(def city : citys) { for(def city : citys) {
def cityCode = cityCodeDataMap.get(city) def cityCode = cityCodeDataMap.get(city)
//
// exportBigdataDealFJW(cityCode, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd"))
//
exportBigdataCommunityGJRH(cityCode) exportBigdataCommunityGJRH(cityCode)
exportBigdataBuildingFJW(cityCode) exportBigdataBuildingFJW(cityCode)
exportBigdataCommfeeGj(cityCode) exportBigdataCommfeeGj(cityCode)
//
exportBigdataDealFJW(cityCode, '2021-11-01')
exportBigdataCommbuildstruGJ(cityCode) exportBigdataCommbuildstruGJ(cityCode)
exportBigdataCommhousetypeGJ(cityCode) exportBigdataCommhousetypeGJ(cityCode)
exportBigdataCommbuildyearGj(cityCode) exportBigdataCommbuildyearGj(cityCode)
@@ -49,6 +52,10 @@ class ExportDistrictRongHe extends BaseBatch{
} }
exportEvaluateRate() exportEvaluateRate()
exportOk() exportOk()
//
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
// SSHHelper jschCommand = new SSHHelper();
// System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/package.sh ${curDate}"));
} }
@@ -319,105 +326,109 @@ class ExportDistrictRongHe extends BaseBatch{
def stop = false def stop = false
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def index = 0 def index = 0
def curStartDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()),"yyyy-MM-dd")
def curSndDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(new Date()),"yyyy-MM-dd")
while(!stop){ while(!stop){
index++ index++
def dealStartStr = startDateStr def curStartStr = curStartDateStr
def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_DEAL_FJW_ADD_${dealStartStr.replace("-","")}_0001.dat" def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
if(CommonDatautils.isWindow()){ if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_DEAL_FJW_ADD_${dealStartStr.replace("-","")}_0001.dat" fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
} }
def dealEndStr = DateUtils.add(startDateStr, 1)
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where CITYID='${cityCode}' and dealdate>=to_date('${dealStartStr}','yyyy-MM-dd') and dealdate<to_date('${dealEndStr}','yyyy-MM-dd')";
def allRows = [] def allRows = []
sqlInstance.eachRow(sql,{r-> if(index==1){
def data = [] def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where CITYID='${cityCode}' and dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";
data << r.CITYID sqlInstance.eachRow(sql,{r->
data << r.DEALID def data = []
data << r.CITYNAME data << r.CITYID
data << r.DISTID data << r.DEALID
data << r.DISTNAME data << r.CITYNAME
data << r.STREET data << r.DISTID
data << r.STREETNO data << r.DISTNAME
data << r.COMMID data << r.STREET
data << r.COMMNAME data << r.STREETNO
data << r.BUILDID data << r.COMMID
data << r.BUILDNO data << r.COMMNAME
data << r.BUILDNAME data << r.BUILDID
data << r.UNITID data << r.BUILDNO
data << r.UNITNO data << r.BUILDNAME
data << r.UNITNAME data << r.UNITID
data << r.HOUSEID data << r.UNITNO
data << r.ADDRESS data << r.UNITNAME
data << r.BUILDYEAR data << r.HOUSEID
data << r.PROPTYPE data << r.ADDRESS
data << r.PROPYEARS data << r.BUILDYEAR
data << r.PERIODNUM data << r.PROPTYPE
data << r.FLOORNUM data << r.PROPYEARS
data << r.FLOORTYPE data << r.PERIODNUM
data << r.FLOORCNT data << r.FLOORNUM
if(r.BUILDTYPE){ data << r.FLOORTYPE
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") data << r.FLOORCNT
data << arrs[0].trim() if(r.BUILDTYPE){
}else{ def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
data << '' data << arrs[0].trim()
} }else{
data << r.BUILDSTRU data << ''
data << r.BUILDAREA }
data << r.LIVINGAREA data << r.BUILDSTRU
data << r.HOUSESTYLE data << r.BUILDAREA
data << r.ROOMCNT data << r.LIVINGAREA
data << r.LIVINGROOMCNT data << r.HOUSESTYLE
data << r.KITCHENCNT data << r.ROOMCNT
data << r.BATHROOMCNT data << r.LIVINGROOMCNT
data << r.HOUSEUSE data << r.KITCHENCNT
data << r.HOUSESTRU data << r.BATHROOMCNT
data << r.HOUSEASPECT data << r.HOUSEUSE
data << r.FITMENT data << r.HOUSESTRU
data << r.EVALUATENO data << r.HOUSEASPECT
data << r.DEALUNITPRICE data << r.FITMENT
data << r.DEALTOTALPRICE data << r.EVALUATENO
data << r.DECLAREPRICE data << r.DEALUNITPRICE
data << r.EVALUATEPRICE data << r.DEALTOTALPRICE
data << r.CONTRACTNO data << r.DECLAREPRICE
data << r.DEALDATE data << r.EVALUATEPRICE
data << r.DEALTYPE data << r.CONTRACTNO
data << r.PAYTYPE data << r.DEALDATE
data << r.DEALPERIOD data << r.DEALTYPE
data << r.IFELEVATOR data << r.PAYTYPE
data << r.COMPAREAGARDEN data << r.DEALPERIOD
data << r.COMPAREATERRACE data << r.IFELEVATOR
data << r.COMPAREABALCONY data << r.COMPAREAGARDEN
data << r.COMPAREABAYWINDOW data << r.COMPAREATERRACE
data << r.COMPAREAATTIC data << r.COMPAREABALCONY
data << r.COMPAREAGARAGE data << r.COMPAREABAYWINDOW
data << r.COMPAREABASEMENT data << r.COMPAREAATTIC
data << r.COMPAREAOTHER data << r.COMPAREAGARAGE
data << r.DATASOURCE data << r.COMPAREABASEMENT
data << r.AGENCYID data << r.COMPAREAOTHER
data << r.AGENCYNAME data << r.DATASOURCE
data << r.REMARK data << r.AGENCYID
data << r.CREATETIME1 data << r.AGENCYNAME
data << r.UPDATETIME1 data << r.REMARK
data << r.SOURCECODE data << r.CREATETIME1
data << r.WORKDATE data << r.UPDATETIME1
allRows << data data << r.SOURCECODE
}); data << r.WORKDATE
allRows << data
if(dealStartStr == endDateStr){ });
}
if(curStartDateStr == curSndDateStr){
stop = true stop = true
} }
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}") CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}")
println "BIGDATA_DEAL_FJW 上传Ftp完成......${dealStartStr}" println "BIGDATA_DEAL_FJW 上传Ftp完成......${curStartStr}"
startDateStr = dealEndStr def curEndStr = DateUtils.add(curStartStr, 1)
curStartDateStr = curEndStr
} }
} }
def exportAllBigdataDealFJW(){ def exportAllBigdataDealFJW(){
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ"; def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where DEALDATE<to_date('2021-12-01','yyyy-MM-dd')";
def allRows = [] def allRows = []
sqlInstance.eachRow(sql,{r-> sqlInstance.eachRow(sql,{r->
def data = [] def data = []
@@ -487,15 +498,15 @@ class ExportDistrictRongHe extends BaseBatch{
data << r.AGENCYNAME data << r.AGENCYNAME
data << r.REMARK data << r.REMARK
data << r.CREATETIME1 data << r.CREATETIME1
data << r.UPDATETIME2 data << r.UPDATETIME1
data << r.SOURCECODE data << r.SOURCECODE
data << r.WORKDATE data << r.WORKDATE
allRows << data allRows << data
}); });
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/All/FJW_000000_All_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){ if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\All\\FJW_000000_All_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
} }
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
println "BIGDATA_DEAL_FJW 导出完成......" println "BIGDATA_DEAL_FJW 导出完成......"
@@ -829,8 +829,6 @@ class CommonDatautils {
keys.put("lat", 1); keys.put("lat", 1);
keys.put("lng", 1); keys.put("lng", 1);
keys.put("name_cat", 1); keys.put("name_cat", 1);
keys.put("name_cat", 1);
keys.put("name_cat", 1);
DSCursor cursor = getDataStoreConnection().find(q, keys); DSCursor cursor = getDataStoreConnection().find(q, keys);
if(cursor){ if(cursor){
for(def obj : cursor){ for(def obj : cursor){
@@ -0,0 +1,144 @@
package com.fangjia.script.fuzhou
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBCursor
import com.mongodb.DBObject
import org.apache.commons.io.FileUtils
/**
* 福州数据结果统计
*/
class FzDataResultStat {
public static final DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName("fangjia", "region_stat", "mongo.db.fangjia.com")
static main(args) {
def task = new FzDataResultStat()
//task.priceMarkupRun();
task.regionPriceTrendRun()
}
def priceMarkupRun(){
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
def map1 = [:]
for(def s : list1){
def row = s.split("@\\|@")
def key = row[2]+"##"+row[3]+"##"+row[4]
map1.put(key, row[5])
}
List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv"))
def map2 = [:]
for(def s : list2) {
def row = s.split("@\\|@")
def key = row[2] + "##" + row[3] + "##" + row[4]
map2.put(key, row[5])
}
def rows = []
map1.each {k,v->
def row = []
def keys = k.split("##")
def lastPrice = map2.get(k)
def markup = null
if(v && lastPrice){
markup = (v - lastPrice)*100.0/lastPrice
}
rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString(), markup]
}
ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据")
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"];
}
def regionPriceTrendRun(){
def city = "福州"
def add = -6;
def months = []
while(add<0){
months << DateUtils.addMonth(new Date(), add)
add++
}
def prices = exportRegion(city, months)
def curMonth = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")
ExcelUtils.write("D:\\数据\\福州\\区域均价半年走势_${curMonth}.xlsx", prices, "数据")
}
def exportRegion(city, months){
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"];
def datas =[]
def header = ["城市", "区域"]
for(def m : months){
header << m.toString().substring(0, m.toString().lastIndexOf("-"))
}
datas << header
for(def r : regions){
def prices = findRegionPrice(city, r, months);
datas << prices
}
return datas;
}
public def findRegionPrice(city, region, weekends){
def prices = []
prices << city
prices << region
for(def weekend : weekends){
def startDate = DateUtils.parseFromTightByPattern(weekend, "yyyy-MM-dd")
def endDate = DateUtils.getLastDayOfMonth(startDate)
def avgPrice = queryCityRegionPrice(city, region, startDate, endDate);
prices << avgPrice
}
return prices;
}
public def queryCityRegionPrice(city, region, startDate, endDate){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
queryCondition.put("region", region);
queryCondition.put("block", null);
queryCondition.put('weekend', new BasicDBObject('$gte', startDate).append('$lte',endDate));
DBCursor cur = REGION_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def price = 0;
while(cur.hasNext()) {
DBObject dbo = cur.next()
price = dbo.get("avg_price");
if(price){
break;
}
}
return price;
}
public def findWeekends(limit){
def index = 1;
def weekends = [];
def date = new Date();
while(index <= limit){
def time = DateUtils.getFirstDayOfWeek(date)
date = time;
weekends << DateUtils.formatDateByPattern(time, "yyyy-MM-dd");
index ++;
}
weekends = weekends.reverse();
return weekends;
}
}
@@ -63,8 +63,7 @@ class DistrictIndexBatch extends BaseBatch{
def run(){ def run(){
def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def citys = CommonDatautils.findAllCity(); def citys = CommonDatautils.findAllCity();
citys.remove("上海") citys = citys[22..(citys.size()-1)]
citys.remove("北京")
for(def city : citys){ for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2"]) def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2"])
// dists = dists[0..20] // dists = dists[0..20]
@@ -367,7 +366,7 @@ class DistrictIndexBatch extends BaseBatch{
def findHouse(def dist, def startDate, def endDate){ def findHouse(def dist, def startDate, def endDate){
def nameQuery = getAllName(dist); def nameQuery = getAllName(dist);
final StringBuffer sqlBuffer = new StringBuffer() final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT area FROM \"seawater-20*\" WHERE 1=1") sqlBuffer.append("SELECT avg(area) as avg_area FROM \"seawater-20*\" WHERE 1=1")
sqlBuffer.append(" and city = '"+dist.city+"'") sqlBuffer.append(" and city = '"+dist.city+"'")
sqlBuffer.append(" and region = '"+dist.region+"'") sqlBuffer.append(" and region = '"+dist.region+"'")
sqlBuffer.append(" and cat = 'sell'") sqlBuffer.append(" and cat = 'sell'")
@@ -375,10 +374,12 @@ class DistrictIndexBatch extends BaseBatch{
sqlBuffer.append(" and (" + nameQuery + ")"); sqlBuffer.append(" and (" + nameQuery + ")");
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def areas = [] def areas = []
def avg_area = null
while (rs.next()){ while (rs.next()){
areas << rs.getDouble("area") // areas << rs.getDouble("area")
avg_area = rs.getDouble("avg_area")
} }
return areas; return avg_area;
} }
def getAllName(dist){ def getAllName(dist){