This commit is contained in:
2022-02-22 09:53:37 +08:00
parent 5c6e8f6f8d
commit 1aa3020361
10 changed files with 485 additions and 156 deletions
@@ -13,8 +13,7 @@ import org.bson.types.ObjectId
*/
class DealPriceDataProcess {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def sqlInstance2 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
@@ -31,16 +30,25 @@ class DealPriceDataProcess {
}
def delDealByPe(){
def yearMonth = 202112
def thirdMonth = 202111
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\估值模型测试202112_1223 - 副本.xlsx",0,1)
def yearMonth = 202201
def thirdMonth = 202112
// def rets = findAllDealByPe(yearMonth, 70)
// for(def r : rets){
// deleteDealPrice(r)
// }
def list = findAllThirdByPe(yearMonth, 70)
for(def r : list){
deletThirdPrice(r, thirdMonth)
}
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\估值模型测试202201_1221 - 副本.xlsx",0,1)
for(def d : datas){
if(d['G']){
def rets = findDealByPe(d['A'], yearMonth, d['G'])
for(def r : rets){
deleteDealPrice(r)
}
}
// if(d['G']){
// def rets = findDealByPe(d['A'], yearMonth, d['G'])
// for(def r : rets){
// deleteDealPrice(r)
// }
// }
// if(d['H']){
// def rets = findCityByPpeRange(d['A'], yearMonth, 15, 20)
@@ -161,6 +169,26 @@ class DealPriceDataProcess {
return count
}
def findAllDealByPe(yearMonth, peVal){
def sql = "select cityid,cityname,distid,commid,commname,unit_price,target_avg_price,pe,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate from avm_evaluation_item_gj where year_month=${yearMonth} and data_type='成交' and pe>${peVal}";
def rets = []
sqlInstance236.eachRow(sql,{r->
def map = [:]
map.cityname = r.cityname
map.cityid=r.cityid
map.distid = r.distid
map.commid = r.commid
map.commname = r.commname
map.deal_price = r.target_avg_price
map.unit_price = r.unit_price
map.buildarea = r.buildarea
map.ppe = r.pe
map.dealdate = r.dealdate
rets << map
})
return rets
}
def findDealByPe(city, yearMonth, peVal){
def sql = "select cityid,cityname,distid,commid,commname,unit_price,target_avg_price,pe,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate from avm_evaluation_item_gj where year_month=${yearMonth} and cityname='${city}' and data_type='成交' and pe>${peVal}";
def rets = []
@@ -181,6 +209,20 @@ class DealPriceDataProcess {
return rets
}
def findAllThirdByPe(yearMonth, peVal){
def sql = "select year_month, commid, target_avg_price from avm_evaluation_item_gj where year_month=${yearMonth} and data_type='第三方' and pe>${peVal}";
def rets = []
sqlInstance236.eachRow(sql,{r->
def map = [:]
map.year_month = r.year_month
map.commid = r.commid
map.target_avg_price = r.target_avg_price
rets << map
})
return rets
}
def findThirdByPe(city, yearMonth, peVal){
def sql = "select year_month, commid, target_avg_price from avm_evaluation_item_gj where year_month=${yearMonth} and cityname='${city}' and data_type='第三方' and pe>${peVal}";
def rets = []
@@ -24,21 +24,21 @@ class ExportOracleData {
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
//不会变
exportEvaluationRateData()
exportBigdataPoiData()
exportBigdataDistrictData()
exportGuidedPrice()//随人工改变
exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策
//处理小区情况
exportBigdataCommunity()
exportCommunityStaging()
exportCommunityBuileType()
// //不会变
// exportEvaluationRateData()
// exportBigdataPoiData()
// exportBigdataDistrictData()
//
// exportGuidedPrice()//随人工改变
// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策
//
// //处理小区情况
// exportBigdataCommunity()
// exportCommunityStaging()
// exportCommunityBuileType()
exportThirdPartyPrice2(lastMonth)
exportBigdataForsale("2021-12-19")
// exportBigdataForsale("2021-12-19")
exportBigdataDeal("2021-12-01")
@@ -879,7 +879,7 @@ class ExportOracleData {
//
// for(def city : citys) {
// def cityCode = cityCodeDataMap.get(city)
def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj";
def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month}";
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
@@ -927,7 +927,7 @@ class ExportOracleData {
allRows << data
});
println allRows.size()
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
// }
println "community_p_describe_gj 数据文件输出完成......"
CcbFtpUtils.ftpUpload(fileName, curDate)
@@ -948,7 +948,7 @@ class ExportOracleData {
for(def city : citys){
def cityCode = cityCodeDataMap.get(city)
def sql = "select ID, CITYID, COMMUNITY_ID, NETLISTING_ID, PRICE_DATE, to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE from comm_case_detail_gj where cityid='${cityCode}'";//price_date=${month} and
def sql = "select ID, CITYID, COMMUNITY_ID, NETLISTING_ID, PRICE_DATE, to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE from comm_case_detail_gj where cityid='${cityCode}' and price_date=${month}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
@@ -224,18 +224,18 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
}
//def fileName = "D:\\数据\\建行\\房产估值\\成交数据_1222 - 副本.xlsx"
//def dealMap = findDealCount(sqlInstance236, "2021-11-01", "2021-12-01")
//def testMap = findCaluateCityPPE(sqlInstance236, 202112)
//ExcelUtils.modify(fileName,{d->
// if(d['C']){
// def oldcount = dealMap.get(d['C'])
// d['R'] = oldcount
// def casecount = testMap.get(d['C'])
// d['S'] = casecount
// println d
// }
//},0)
def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx"
def dealMap = findDealCount(sqlInstance236, "2021-12-01", "2022-01-01")
def testMap = findCaluateCityPPE(sqlInstance236, 202201)
ExcelUtils.modify(fileName,{d->
if(d['C']){
def oldcount = dealMap.get(d['C'])
d['T'] = oldcount
def casecount = testMap.get(d['C'])
d['U'] = casecount
println d
}
},0)
//ExcelUtils.multiThreadingModify("D:\\数据\\小区\\5个城市小区列表0723(1).xlsx",{d->
// if(d['I'] && d['K']){
@@ -39,8 +39,8 @@ class TradeStatLog {
static main(args) {
def task = new TradeStatLog()
task.run()//统计最近3个月抓取的成交数据量
// task.exportTrade()//导出未入库成交
// task.run()//统计最近3个月抓取的成交数据量
task.exportTrade()//导出未入库成交
// task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
// task.averageStat()
@@ -129,7 +129,7 @@ class TradeStatLog {
}
ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\12月成交统计.xlsx", rets, "数据")
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\12月成交统计.xlsx", rets, "数据")
println "表格生成完成"
}
@@ -479,16 +479,18 @@ class TradeStatLog {
def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"]
def rets = []
rets << headers
def startDate = "2021-12-01"
def endDate = "2022-01-01"
for(def city : citys){
def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern("2021-10-01","yyyy-MM-dd"), DateUtils.parseFromTightByPattern("2021-11-01","yyyy-MM-dd"))
def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
// def trades = getYouDaData()
//def statMap = getDistrictStatByCity(city, weekend)
def nameMap = getAllDistrict(city)
def rMap = findDealIds(city);
def rMap = findDealIds(city, startDate, endDate);
def ids = rMap.ids
def details = rMap.detail
@@ -559,7 +561,7 @@ class TradeStatLog {
}
println city +" "+rets.size()
}
ExcelUtils.write("d:\\数据\\成交未入库记录(10月).xlsx", rets,"数据")
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(12月).xlsx", rets,"数据")
}
def deleteDistrict(){
@@ -615,8 +617,8 @@ class TradeStatLog {
return trades
}
def findDealIds(city){
def sql = "select * from bigdata_deal_gj where dealdate>=to_date('2021-10-01','yyyy-mm-dd') and dealdate<to_date('2021-11-01','yyyy-mm-dd') and cityname='${city}'";//and dealdate<to_date('2021-08-01','yyyy-mm-dd')
def findDealIds(city, startDate, endDate){
def sql = "select * from bigdata_deal_gj where dealdate>=to_date('${startDate}','yyyy-mm-dd') and dealdate<to_date('${endDate}','yyyy-mm-dd') and cityname='${city}'";//and dealdate<to_date('2021-08-01','yyyy-mm-dd')
def ids = []
def detailInfos = []
sqlInstance.eachRow(sql,{r->
@@ -11,10 +11,14 @@ import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\人工核实_8511.xlsx", { d ->
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
// if(newRegion && !d['K']) {
@@ -332,7 +336,7 @@ def city = "
//def rets = []
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\国信达名字加入映射别名.xlsx", { d ->
// if(d['A']){
// if(d['A'] && d['E'].toString().contains("鑫康花园北区")){
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
// if(newRegion){
// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
@@ -357,7 +361,7 @@ def city = "
//ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\新增映射别名.xlsx", rets,"数据")
def rets = []
//def rets = []
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\查询别名表,将国信达数据增加到映射别名表里.xlsx", { d ->
// if(d['A']){
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
@@ -388,15 +392,64 @@ def rets = []
//ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\新增映射别名_2.xlsx", rets,"数据")
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\修改状态的数据.xlsx", { d ->
if(d['A']) {
def newRegion = RegionData.formatRegion(d['A'], d['B'])
if (newRegion) {
def dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['C'], "district")
if (dist) {
rets << [dist._id.toString(), d['C']]
}
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\修改状态的数据.xlsx", { d ->
// if(d['A']) {
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
// if (newRegion) {
// def dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['C'], "district")
// if (dist) {
// rets << [dist._id.toString(), d['C']]
// }
// }
// }
//})
//ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\映射别名存在——修改状态.xlsx", rets,"数据")
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\修改后结果数据.xlsx",{d->
// if(d['A']){
// def dist = CommonDatautils.findDistrictById(d['B'])
// if(dist){
// d['E'] = dist.city
// d['F'] = dist.region
// d['G'] = dist.name
// d['H'] = dist.estate_type2
// d['I'] = dist.name_cat
// d['J'] = dist.status
// }
// }
//})
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\修改后结果数据.xlsx",{d->
// if(d['B']) {
// def sql = "select count(*) as count from bigdata_community_gj where commid='${d['B']}'";
// def count = 0;
// sqlInstance.eachRow(sql, { r ->
// count = r.count
// })
// if(count>=1){
// d['J'] = "oracle存在"
// }
// }
//})
def index = 0
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\漏映射别名.xlsx", { d ->
if (d['B']) {
def dist = CommonDatautils.findDistrictFieldById(d['B'],["city","region","name","mapping_alias"])
if(dist){
def alias = dist.mapping_alias
if(!alias){
alias = []
}
if(alias && !alias.contains(d['C'])){
index++
println index
alias << d['C']
def udata = new BasicDBObject()
udata.put("mapping_alias",alias)
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
}
}
}
})
ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\映射别名存在——修改状态.xlsx", rets,"数据")
})
@@ -0,0 +1,78 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.bson.types.ObjectId
class MatchDealPriceRegion {
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
static main(args) {
def task = new MatchDealPriceRegion();
task.run()
}
def run(){
def index = 0
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\成交\\部分未匹配成交.xlsx",{d->
def newRegion = null
if(d['C']){
newRegion = RegionData.formatRegion(d['B'], d['C'])
}
if(!newRegion){
index++
newRegion = findRegionByThirdPrice(d['B'], d['D']);
if(!newRegion || (newRegion && newRegion==d['C'])){
def rets = CommonDatautils.findDistrictsByName(d['B'], d['D'],"district")
if(rets && rets.size()==1){
newRegion = rets[0].region
}
}
if(newRegion && newRegion!=d['C']){
def query = new BasicDBObject()
query.put("_id", new ObjectId(d['A']))
def data = new BasicDBObject()
data.put("region", newRegion)
data.put("m_date", new Date())
println index+" "+d['A']+" "+d['C']+" "+newRegion
DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true)
}
}
})
}
def findRegionByThirdPrice(def city, def name){
def q = new BasicDBObject()
q.put("city", city)
q.put("name", name)
def dbcursor = collection.find(q).sort(new BasicDBObject("s_date",-1)).limit(10)
def map = [:]
for(def d : dbcursor){
def count = map.get(d.region)
if(!count){
count = 0
}
map.put(d.region, count+1)
}
if(map){
if(map.size()>1){
map = map.sort(){ a, b ->
b.value.compareTo(a.value)
}
}
def key = map.keySet().getAt(0)
return key
}
return null
}
}
@@ -27,15 +27,18 @@ class ExportDistrictRongHe extends BaseBatch{
def citys = CcbCommonUtils.citys;
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
//检查ok文件是否上传
// //检查ok文件是否上传
// exportAllBigdataDealFJW()
for(def city : citys) {
def cityCode = cityCodeDataMap.get(city)
//
// exportBigdataDealFJW(cityCode, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd"))
//
exportBigdataCommunityGJRH(cityCode)
exportBigdataBuildingFJW(cityCode)
exportBigdataCommfeeGj(cityCode)
exportBigdataDealFJW(cityCode, '2021-11-01')
//
exportBigdataCommbuildstruGJ(cityCode)
exportBigdataCommhousetypeGJ(cityCode)
exportBigdataCommbuildyearGj(cityCode)
@@ -49,6 +52,10 @@ class ExportDistrictRongHe extends BaseBatch{
}
exportEvaluateRate()
exportOk()
//
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
// SSHHelper jschCommand = new SSHHelper();
// System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/package.sh ${curDate}"));
}
@@ -319,105 +326,109 @@ class ExportDistrictRongHe extends BaseBatch{
def stop = false
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def index = 0
def curStartDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()),"yyyy-MM-dd")
def curSndDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(new Date()),"yyyy-MM-dd")
while(!stop){
index++
def dealStartStr = startDateStr
def curStartStr = curStartDateStr
def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_DEAL_FJW_ADD_${dealStartStr.replace("-","")}_0001.dat"
def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_DEAL_FJW_ADD_${dealStartStr.replace("-","")}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
}
def dealEndStr = DateUtils.add(startDateStr, 1)
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where CITYID='${cityCode}' and dealdate>=to_date('${dealStartStr}','yyyy-MM-dd') and dealdate<to_date('${dealEndStr}','yyyy-MM-dd')";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.DEALID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.STREET
data << r.STREETNO
data << r.COMMID
data << r.COMMNAME
data << r.BUILDID
data << r.BUILDNO
data << r.BUILDNAME
data << r.UNITID
data << r.UNITNO
data << r.UNITNAME
data << r.HOUSEID
data << r.ADDRESS
data << r.BUILDYEAR
data << r.PROPTYPE
data << r.PROPYEARS
data << r.PERIODNUM
data << r.FLOORNUM
data << r.FLOORTYPE
data << r.FLOORCNT
if(r.BUILDTYPE){
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.BUILDSTRU
data << r.BUILDAREA
data << r.LIVINGAREA
data << r.HOUSESTYLE
data << r.ROOMCNT
data << r.LIVINGROOMCNT
data << r.KITCHENCNT
data << r.BATHROOMCNT
data << r.HOUSEUSE
data << r.HOUSESTRU
data << r.HOUSEASPECT
data << r.FITMENT
data << r.EVALUATENO
data << r.DEALUNITPRICE
data << r.DEALTOTALPRICE
data << r.DECLAREPRICE
data << r.EVALUATEPRICE
data << r.CONTRACTNO
data << r.DEALDATE
data << r.DEALTYPE
data << r.PAYTYPE
data << r.DEALPERIOD
data << r.IFELEVATOR
data << r.COMPAREAGARDEN
data << r.COMPAREATERRACE
data << r.COMPAREABALCONY
data << r.COMPAREABAYWINDOW
data << r.COMPAREAATTIC
data << r.COMPAREAGARAGE
data << r.COMPAREABASEMENT
data << r.COMPAREAOTHER
data << r.DATASOURCE
data << r.AGENCYID
data << r.AGENCYNAME
data << r.REMARK
data << r.CREATETIME1
data << r.UPDATETIME1
data << r.SOURCECODE
data << r.WORKDATE
allRows << data
});
if(dealStartStr == endDateStr){
if(index==1){
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where CITYID='${cityCode}' and dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.DEALID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.STREET
data << r.STREETNO
data << r.COMMID
data << r.COMMNAME
data << r.BUILDID
data << r.BUILDNO
data << r.BUILDNAME
data << r.UNITID
data << r.UNITNO
data << r.UNITNAME
data << r.HOUSEID
data << r.ADDRESS
data << r.BUILDYEAR
data << r.PROPTYPE
data << r.PROPYEARS
data << r.PERIODNUM
data << r.FLOORNUM
data << r.FLOORTYPE
data << r.FLOORCNT
if(r.BUILDTYPE){
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.BUILDSTRU
data << r.BUILDAREA
data << r.LIVINGAREA
data << r.HOUSESTYLE
data << r.ROOMCNT
data << r.LIVINGROOMCNT
data << r.KITCHENCNT
data << r.BATHROOMCNT
data << r.HOUSEUSE
data << r.HOUSESTRU
data << r.HOUSEASPECT
data << r.FITMENT
data << r.EVALUATENO
data << r.DEALUNITPRICE
data << r.DEALTOTALPRICE
data << r.DECLAREPRICE
data << r.EVALUATEPRICE
data << r.CONTRACTNO
data << r.DEALDATE
data << r.DEALTYPE
data << r.PAYTYPE
data << r.DEALPERIOD
data << r.IFELEVATOR
data << r.COMPAREAGARDEN
data << r.COMPAREATERRACE
data << r.COMPAREABALCONY
data << r.COMPAREABAYWINDOW
data << r.COMPAREAATTIC
data << r.COMPAREAGARAGE
data << r.COMPAREABASEMENT
data << r.COMPAREAOTHER
data << r.DATASOURCE
data << r.AGENCYID
data << r.AGENCYNAME
data << r.REMARK
data << r.CREATETIME1
data << r.UPDATETIME1
data << r.SOURCECODE
data << r.WORKDATE
allRows << data
});
}
if(curStartDateStr == curSndDateStr){
stop = true
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}")
println "BIGDATA_DEAL_FJW 上传Ftp完成......${dealStartStr}"
startDateStr = dealEndStr
println "BIGDATA_DEAL_FJW 上传Ftp完成......${curStartStr}"
def curEndStr = DateUtils.add(curStartStr, 1)
curStartDateStr = curEndStr
}
}
def exportAllBigdataDealFJW(){
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ";
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where DEALDATE<to_date('2021-12-01','yyyy-MM-dd')";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
@@ -487,15 +498,15 @@ class ExportDistrictRongHe extends BaseBatch{
data << r.AGENCYNAME
data << r.REMARK
data << r.CREATETIME1
data << r.UPDATETIME2
data << r.UPDATETIME1
data << r.SOURCECODE
data << r.WORKDATE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/All/FJW_000000_All_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\All\\FJW_000000_All_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
println "BIGDATA_DEAL_FJW 导出完成......"
@@ -829,8 +829,6 @@ class CommonDatautils {
keys.put("lat", 1);
keys.put("lng", 1);
keys.put("name_cat", 1);
keys.put("name_cat", 1);
keys.put("name_cat", 1);
DSCursor cursor = getDataStoreConnection().find(q, keys);
if(cursor){
for(def obj : cursor){
@@ -0,0 +1,144 @@
package com.fangjia.script.fuzhou
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBCursor
import com.mongodb.DBObject
import org.apache.commons.io.FileUtils
/**
* 福州数据结果统计
*/
class FzDataResultStat {
public static final DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName("fangjia", "region_stat", "mongo.db.fangjia.com")
static main(args) {
def task = new FzDataResultStat()
//task.priceMarkupRun();
task.regionPriceTrendRun()
}
def priceMarkupRun(){
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
def map1 = [:]
for(def s : list1){
def row = s.split("@\\|@")
def key = row[2]+"##"+row[3]+"##"+row[4]
map1.put(key, row[5])
}
List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv"))
def map2 = [:]
for(def s : list2) {
def row = s.split("@\\|@")
def key = row[2] + "##" + row[3] + "##" + row[4]
map2.put(key, row[5])
}
def rows = []
map1.each {k,v->
def row = []
def keys = k.split("##")
def lastPrice = map2.get(k)
def markup = null
if(v && lastPrice){
markup = (v - lastPrice)*100.0/lastPrice
}
rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString(), markup]
}
ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据")
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"];
}
def regionPriceTrendRun(){
def city = "福州"
def add = -6;
def months = []
while(add<0){
months << DateUtils.addMonth(new Date(), add)
add++
}
def prices = exportRegion(city, months)
def curMonth = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")
ExcelUtils.write("D:\\数据\\福州\\区域均价半年走势_${curMonth}.xlsx", prices, "数据")
}
def exportRegion(city, months){
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"];
def datas =[]
def header = ["城市", "区域"]
for(def m : months){
header << m.toString().substring(0, m.toString().lastIndexOf("-"))
}
datas << header
for(def r : regions){
def prices = findRegionPrice(city, r, months);
datas << prices
}
return datas;
}
public def findRegionPrice(city, region, weekends){
def prices = []
prices << city
prices << region
for(def weekend : weekends){
def startDate = DateUtils.parseFromTightByPattern(weekend, "yyyy-MM-dd")
def endDate = DateUtils.getLastDayOfMonth(startDate)
def avgPrice = queryCityRegionPrice(city, region, startDate, endDate);
prices << avgPrice
}
return prices;
}
public def queryCityRegionPrice(city, region, startDate, endDate){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
queryCondition.put("region", region);
queryCondition.put("block", null);
queryCondition.put('weekend', new BasicDBObject('$gte', startDate).append('$lte',endDate));
DBCursor cur = REGION_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def price = 0;
while(cur.hasNext()) {
DBObject dbo = cur.next()
price = dbo.get("avg_price");
if(price){
break;
}
}
return price;
}
public def findWeekends(limit){
def index = 1;
def weekends = [];
def date = new Date();
while(index <= limit){
def time = DateUtils.getFirstDayOfWeek(date)
date = time;
weekends << DateUtils.formatDateByPattern(time, "yyyy-MM-dd");
index ++;
}
weekends = weekends.reverse();
return weekends;
}
}
@@ -63,8 +63,7 @@ class DistrictIndexBatch extends BaseBatch{
def run(){
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def citys = CommonDatautils.findAllCity();
citys.remove("上海")
citys.remove("北京")
citys = citys[22..(citys.size()-1)]
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2"])
// dists = dists[0..20]
@@ -367,7 +366,7 @@ class DistrictIndexBatch extends BaseBatch{
def findHouse(def dist, def startDate, def endDate){
def nameQuery = getAllName(dist);
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT area FROM \"seawater-20*\" WHERE 1=1")
sqlBuffer.append("SELECT avg(area) as avg_area FROM \"seawater-20*\" WHERE 1=1")
sqlBuffer.append(" and city = '"+dist.city+"'")
sqlBuffer.append(" and region = '"+dist.region+"'")
sqlBuffer.append(" and cat = 'sell'")
@@ -375,10 +374,12 @@ class DistrictIndexBatch extends BaseBatch{
sqlBuffer.append(" and (" + nameQuery + ")");
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def areas = []
def avg_area = null
while (rs.next()){
areas << rs.getDouble("area")
// areas << rs.getDouble("area")
avg_area = rs.getDouble("avg_area")
}
return areas;
return avg_area;
}
def getAllName(dist){