update
This commit is contained in:
+2
-1
@@ -82,9 +82,10 @@ class CcbCommonUtils {
|
||||
return rets
|
||||
}
|
||||
|
||||
public static def findGuidePrice(city){
|
||||
public static def findGuidePrice(city, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put('c_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt', DateUtils.parseFromTightYYYYMMDD(endDate)));
|
||||
def dbcursor = DISTRICT_GUIDE_PRICE_COLL.find(q)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
|
||||
+1
-1
@@ -25,7 +25,7 @@ class DistrictGuidePriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findGuidePrice(city)
|
||||
def prices = CcbCommonUtils.findGuidePrice(city, "2022-01-1", "2022-02-1")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||
|
||||
+33
-31
@@ -24,12 +24,14 @@ class ExportOracleData {
|
||||
}
|
||||
|
||||
def run() {
|
||||
//
|
||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
|
||||
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
// exportGuidedPrice()
|
||||
// exportCommunityGuidedPrice()
|
||||
exportCommunityGuidedPrice(lastMonth.toString()+"28")
|
||||
//
|
||||
// //exportThirdPartyPrice()//废弃掉,老表
|
||||
// exportThirdPartyPrice2(202111)
|
||||
// exportThirdPartyPrice2(lastMonth)
|
||||
//
|
||||
//
|
||||
//
|
||||
@@ -40,20 +42,20 @@ class ExportOracleData {
|
||||
// exportCommunityStaging()
|
||||
// exportCommunityBuileType()
|
||||
// exportBigdataDistrictData()
|
||||
exportBigdataCommunity()
|
||||
// exportBigdataCommunity()
|
||||
// exportBigdataDeal("2021-11-01")
|
||||
|
||||
// exportAvmEvaluationGJNB(202112)
|
||||
// exportAVMEvaluationItemGJNB(202112)
|
||||
// exportCommunityBasePrice(202112)
|
||||
// exportAvmEvaluationGJNB(curMonth)
|
||||
// exportAVMEvaluationItemGJNB(curMonth)
|
||||
// exportCommunityBasePrice(curMonth)
|
||||
//
|
||||
// exportCityPrice(202112)
|
||||
// exportRegionPrice(202112)
|
||||
// exportCityPrice(curMonth)
|
||||
// exportRegionPrice(curMonth)
|
||||
|
||||
// exportCommunityBasePriceGj(202112)
|
||||
// exportCommunityBasePriceGj(curMonth)
|
||||
|
||||
// exportCommunityPdescribeGj(202112)
|
||||
// exportCommCaseDetailGj(202112)
|
||||
// exportCommunityPdescribeGj(curMonth)
|
||||
// exportCommCaseDetailGj(curMonth)
|
||||
// exportLastFile()
|
||||
}
|
||||
|
||||
@@ -255,26 +257,26 @@ class ExportOracleData {
|
||||
println "GUIDED_PRICE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportCommunityGuidedPrice() {
|
||||
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj";
|
||||
def exportCommunityGuidedPrice(def date) {
|
||||
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where createtime>=to_date('${date}','yyyyMMdd')";
|
||||
def allRows = []
|
||||
// sqlInstance.eachRow(sql, { r ->
|
||||
// def data = []
|
||||
// data << r.ID
|
||||
// data << r.COMMID
|
||||
// data << r.COMMNAME
|
||||
// data << r.CITYID
|
||||
// data << r.CITYNAME
|
||||
// data << r.DISTID
|
||||
// data << r.DISTNAME
|
||||
// data << r.AVGPRICE
|
||||
// data << r.SOURCETYPE
|
||||
// data << r.REMARK
|
||||
// data << r.CREATETIME
|
||||
// data << r.UPDATETIME
|
||||
// data << r.SOURCECODE
|
||||
// allRows << data
|
||||
// });
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.CITYID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.AVGPRICE
|
||||
data << r.SOURCETYPE
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
|
||||
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
|
||||
+1
-1
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2021-10-1", "2021-11-1")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-1", "2022-02-1")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||
|
||||
+26
-6
@@ -4,6 +4,8 @@ import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
|
||||
class LoadGuide {
|
||||
public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017)
|
||||
@@ -14,13 +16,31 @@ class LoadGuide {
|
||||
}
|
||||
|
||||
def run(){
|
||||
// match("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
|
||||
|
||||
// saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞指导价218.xlsx")
|
||||
// saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞二手房政府指导价.xls")
|
||||
// saveChengduoPrice("D:\\数据\\建行\\房产估值\\指导价\\成都部分住宅小区二手房成交参考价格201-286-258.xlsx")
|
||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\绍兴指导价小区_0909.xlsx")
|
||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\宁波交易参考价格.xlsx")
|
||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安成交参考价格.xlsx")
|
||||
savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
|
||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\无锡100 广州96 二手住房成交参考价格表.xlsx")
|
||||
savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx")
|
||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx")
|
||||
|
||||
}
|
||||
def match(def filename){
|
||||
def city = "西安"
|
||||
ExcelUtils.modify(filename,{d->
|
||||
def newRegion = RegionData.formatRegion(city, d['A'])
|
||||
if(newRegion){
|
||||
def rMap = SeaweedDataMatchProcess.doMatch(city, newRegion, d['B'], null);
|
||||
if(rMap){
|
||||
d['D'] = city
|
||||
d['E'] = rMap.region
|
||||
d['F'] = rMap.name
|
||||
}
|
||||
}
|
||||
},0)
|
||||
|
||||
}
|
||||
|
||||
@@ -32,8 +52,8 @@ class LoadGuide {
|
||||
def o = new BasicDBObject()
|
||||
o.city = d['E']
|
||||
o.region = d['A']
|
||||
o.name = d['C']
|
||||
o.avg_price = d['D'].toString().toDouble().toInteger()
|
||||
o.name = d['B']
|
||||
o.avg_price = d['C'].toString().toDouble().toInteger()
|
||||
o.match_city = d['E']
|
||||
o.match_region = d['F']
|
||||
o.match_name = d['G']
|
||||
@@ -57,11 +77,11 @@ class LoadGuide {
|
||||
o.city = d['E']
|
||||
o.region = d['A']
|
||||
o.name = d['B']
|
||||
o.avg_price = d['D'].toString().toDouble().toInteger()
|
||||
o.avg_price = d['C'].toString().toDouble().toInteger()
|
||||
o.match_city = d['E']
|
||||
o.match_region = d['F']
|
||||
o.match_name = d['G']
|
||||
o.house_count = d['C'].toString().toDouble().toInteger()
|
||||
// o.house_count = d['C'].toString().toDouble().toInteger()
|
||||
o.source_type = 1 //1:成交参考价 2:参考价
|
||||
o.c_date = new Date()
|
||||
rets << o
|
||||
|
||||
+241
-39
@@ -4,7 +4,107 @@ import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
|
||||
DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d ->
|
||||
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
// if(newRegion && !d['K']) {
|
||||
// def rMap = SeaweedDataMatchProcess.doMatch(d['A'], newRegion, d['C'], d['D']+d['E']);
|
||||
// if(!rMap && d['G']){
|
||||
// rMap = SeaweedDataMatchProcess.doMatch(d['A'], newRegion, d['G'], d['H']);
|
||||
// }
|
||||
// if(rMap){
|
||||
// d['O'] = rMap.city
|
||||
// d['P'] = rMap.region
|
||||
// d['Q'] = rMap.name
|
||||
// d['R'] = rMap.address
|
||||
// }
|
||||
// }
|
||||
//});
|
||||
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\高德反查数据3_2003.xlsx",{d->
|
||||
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
// if(newRegion && !d['K']){
|
||||
//
|
||||
// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'],"district")
|
||||
// if(dist){
|
||||
// d['K']="完全一致"
|
||||
// d['L'] = dist.estate_type2
|
||||
// d['M'] = dist.name_cat
|
||||
// }else{
|
||||
// dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['C'],"district")
|
||||
// if(dist){
|
||||
// d['K']="映射别名"
|
||||
// d['L'] = dist.estate_type2
|
||||
// d['M'] = dist.name_cat
|
||||
// }else{
|
||||
// def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['C'],"district")
|
||||
// if(rets){
|
||||
// dist = rets[0]
|
||||
// }
|
||||
// if(rets){
|
||||
// d['K']="别名"
|
||||
// d['L'] = dist.estate_type2
|
||||
// d['M'] = dist.name_cat
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//},0)
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\高德反查数据3_2003.xlsx",{d->
|
||||
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
// if(newRegion && !d['K'] && d['G']){
|
||||
//
|
||||
// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['G'],"district")
|
||||
// if(dist){
|
||||
// d['K']="完全一致(高德)"
|
||||
// d['L'] = dist.estate_type2
|
||||
// d['M'] = dist.name_cat
|
||||
// }else{
|
||||
// dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['G'],"district")
|
||||
// if(dist){
|
||||
// d['K']="映射别名(高德)"
|
||||
// d['L'] = dist.estate_type2
|
||||
// d['M'] = dist.name_cat
|
||||
// }else{
|
||||
// def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['G'],"district")
|
||||
// if(rets){
|
||||
// dist = rets[0]
|
||||
// }
|
||||
// if(rets){
|
||||
// d['K']="别名(高德)"
|
||||
// d['L'] = dist.estate_type2
|
||||
// d['M'] = dist.name_cat
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//},0)
|
||||
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\价格过滤小区融合情况_1209.xlsx",{d->
|
||||
// def dist = CommonDatautils.findDistrictFieldById(d['F'], ["address","lat","lng"])
|
||||
// if(dist){
|
||||
// d['Z'] = dist.address
|
||||
// if(dist.lat){
|
||||
// d['AA'] = dist.lat/10000000000.0
|
||||
// d['AB'] = dist.lng/10000000000.0
|
||||
// }
|
||||
// }
|
||||
//},0)
|
||||
|
||||
//def cityDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\小区\\重庆高德反查数据_1529.xlsx",0,1)
|
||||
//def cityDataMap = [:]
|
||||
@@ -74,54 +174,156 @@ def city = "
|
||||
//},0)
|
||||
|
||||
|
||||
//ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例.xlsx",{d->
|
||||
//ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例(合并).xlsx",{d->
|
||||
// def obj = CommonDatautils.findDistrict("广州", d['C'], d['D'], "district")
|
||||
// if(obj){
|
||||
// def year = CommonUtils.getDistrictAge(obj)
|
||||
// if(year){
|
||||
// d['H'] = year
|
||||
// }
|
||||
// if(obj.alias){
|
||||
// d['K'] = obj.alias.join(" ")
|
||||
//// def year = CommonUtils.getDistrictAge(obj)
|
||||
//// if(year){
|
||||
//// d['H'] = year
|
||||
//// }
|
||||
//// if(obj.alias){
|
||||
//// d['K'] = obj.alias.join(" ")
|
||||
//// }
|
||||
// if(obj.mapping_alias){
|
||||
// d['X'] = obj.mapping_alias.join(" ")
|
||||
// }
|
||||
//
|
||||
// }
|
||||
//},0)
|
||||
|
||||
def data1s = ExcelUtils.read("D:\\数据\\广发\\图片转换汇总1.8.xlsx",0, 1)
|
||||
def scoreMap = [:]
|
||||
for(def d : data1s){
|
||||
scoreMap.put(d['B'], d)
|
||||
}
|
||||
//def data1s = ExcelUtils.read("D:\\数据\\广发\\图片转换汇总1.8.xlsx",0, 1)
|
||||
//def scoreMap = [:]
|
||||
//for(def d : data1s){
|
||||
// scoreMap.put(d['B'], d)
|
||||
//}
|
||||
//
|
||||
//def data2s = ExcelUtils.read("D:\\数据\\广发\\图片转换汇总1.8.xlsx",1, 1)
|
||||
//def levelMap = [:]
|
||||
//for(def d : data2s){
|
||||
// levelMap.put(d['E'], d)
|
||||
//}
|
||||
//
|
||||
//ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例.xlsx",{d->
|
||||
// if(d['E']){
|
||||
// def row = scoreMap.get(d['E'])
|
||||
// if(row){
|
||||
// d['I'] = row['G']
|
||||
// d['J'] = row['H']
|
||||
// d['K'] = row['I']
|
||||
// d['L'] = row['J']
|
||||
// d['M'] = row['K']
|
||||
// d['N'] = row['L']
|
||||
// d['O'] = row['M']
|
||||
// }
|
||||
//
|
||||
// def data = levelMap.get(d['E'])
|
||||
// if(data){
|
||||
// d['P'] = data['G']
|
||||
// d['Q'] = data['H']
|
||||
// d['R'] = data['I']
|
||||
// d['S'] = data['J']
|
||||
// d['T'] = data['K']
|
||||
// d['U'] = data['L']
|
||||
// d['V'] = data['M']
|
||||
// }
|
||||
// }
|
||||
//},0)
|
||||
|
||||
def data2s = ExcelUtils.read("D:\\数据\\广发\\图片转换汇总1.8.xlsx",1, 1)
|
||||
def levelMap = [:]
|
||||
for(def d : data2s){
|
||||
levelMap.put(d['E'], d)
|
||||
}
|
||||
//ExcelUtils.modify("D:\\数据\\成交\\福州成交.xlsx",{d->
|
||||
// def o = new BasicDBObject()
|
||||
// o.area = d['D'].toDouble()
|
||||
// o.avg_price = d['F'].toInteger()
|
||||
// o.city = "福州"
|
||||
// o.district_name = d['C']
|
||||
// o.region = d['B']
|
||||
// o.source = "贝壳找房"
|
||||
// o.trade_date = DateUtils.parseFromTightYYYYMMDD(d['M'])
|
||||
// o.total_price = d['E'].toInteger()*10000
|
||||
// if(d['I']){
|
||||
// o.hall = d['I'].toInteger()
|
||||
// }
|
||||
//
|
||||
// o.m_date = new Date()
|
||||
// if(d['H']){
|
||||
// o.room = d['H'].toInteger()
|
||||
// }
|
||||
//
|
||||
// o.create_date = new Date()
|
||||
// o.status = 2
|
||||
// DEAL_PRICE.save(o)
|
||||
//
|
||||
//},0)
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例.xlsx",{d->
|
||||
if(d['E']){
|
||||
def row = scoreMap.get(d['E'])
|
||||
if(row){
|
||||
d['I'] = row['G']
|
||||
d['J'] = row['H']
|
||||
d['K'] = row['I']
|
||||
d['L'] = row['J']
|
||||
d['M'] = row['K']
|
||||
d['N'] = row['L']
|
||||
d['O'] = row['M']
|
||||
}
|
||||
|
||||
def data = levelMap.get(d['E'])
|
||||
if(data){
|
||||
d['P'] = data['G']
|
||||
d['Q'] = data['H']
|
||||
d['R'] = data['I']
|
||||
d['S'] = data['J']
|
||||
d['T'] = data['K']
|
||||
d['U'] = data['L']
|
||||
d['V'] = data['M']
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d ->
|
||||
// if(d['K'] && d['K']=="别名"){
|
||||
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
// if(newRegion){
|
||||
// def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['C'],"district")
|
||||
// def dist = null
|
||||
// if(rets){
|
||||
// dist = rets[0]
|
||||
// }
|
||||
// if(dist){
|
||||
// def alias = dist.mapping_alias
|
||||
// if(!alias){
|
||||
// alias = []
|
||||
// }
|
||||
// if(!alias.contains(d['C'])){
|
||||
// alias << d['C']
|
||||
// }
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("mapping_alias",alias)
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//});
|
||||
|
||||
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d ->
|
||||
// if(d['K'] && d['K']=="映射别名(高德)" && d['G']){
|
||||
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
// if(newRegion){
|
||||
// dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['G'],"district")
|
||||
// if(dist){
|
||||
// def alias = dist.mapping_alias
|
||||
// if(!alias){
|
||||
// alias = []
|
||||
// }
|
||||
// if(!alias.contains(d['C'])){
|
||||
// alias << d['C']
|
||||
// }
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("mapping_alias",alias)
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//});
|
||||
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d ->
|
||||
if(d['K'] && d['K']=="别名(高德)" && d['G']){
|
||||
def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
if(newRegion){
|
||||
def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['G'],"district")
|
||||
def dist = null
|
||||
if(rets){
|
||||
dist = rets[0]
|
||||
}
|
||||
if(dist){
|
||||
def alias = dist.mapping_alias
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(d['C'])){
|
||||
alias << d['C']
|
||||
}
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("mapping_alias",alias)
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
});
|
||||
|
||||
+12
@@ -27,6 +27,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
//检查ok文件是否上传
|
||||
// exportAllBigdataDealFJW()
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeDataMap.get(city)
|
||||
@@ -47,6 +48,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
exportBigdataCommlayoutpicFjw(cityCode)
|
||||
}
|
||||
exportEvaluateRate()
|
||||
exportOk()
|
||||
|
||||
}
|
||||
|
||||
@@ -593,4 +595,14 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportOk(){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok"
|
||||
}
|
||||
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
|
||||
CcbFtpUtils.ftpUpload(dataCodeFileName, curDate)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+28
-1
@@ -13,7 +13,34 @@ class DownPdfCheck {
|
||||
|
||||
static main(args) {
|
||||
def task = new DownPdfCheck();
|
||||
task.run();
|
||||
// task.run();
|
||||
|
||||
task.czLogRun()
|
||||
}
|
||||
|
||||
def czLogRun(){
|
||||
def files = getAllFiles("D:\\数据\\常州")
|
||||
if(files){
|
||||
for(def filepath : files){
|
||||
read(filepath);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
def read(file){
|
||||
// File file = new File(fileName);
|
||||
def index = 0
|
||||
|
||||
file.eachLine{it->
|
||||
if(it.contains("4ead0347b4055be6989e726e")||it.contains("61a2e0")){
|
||||
index++
|
||||
|
||||
println it
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def run(){
|
||||
|
||||
+28
@@ -8,6 +8,7 @@ import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.jusfoun.JusfounCommon
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import org.apache.commons.codec.digest.DigestUtils
|
||||
import org.apache.commons.io.FileUtils
|
||||
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
@@ -79,6 +80,14 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
def lastDate = new Date()
|
||||
def startDate = DateUtils.getFirstDayOfMonth(lastDate, -1)
|
||||
|
||||
// List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202111.csv"))
|
||||
// def map2 = [:]
|
||||
// for(def s : list2) {
|
||||
// def row = s.split("@\\|@")
|
||||
// def key = row[2] + "##" + row[3] + "##" + row[4]
|
||||
// map2.put(key, row[5])
|
||||
// }
|
||||
|
||||
def result = []
|
||||
for(def rMap : districts){
|
||||
def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate)
|
||||
@@ -90,6 +99,25 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
m << rMap.region
|
||||
m << rMap.name
|
||||
m << dbo.get("avg_price")
|
||||
// def lastMonthPrice = map2.get(rMap.city + "##" + rMap.region+"##"+rMap.name)
|
||||
// if(lastMonthPrice && dbo.get("avg_price")){
|
||||
// lastMonthPrice = lastMonthPrice.toInteger()
|
||||
// def markup = (dbo.get("avg_price")-lastMonthPrice)*100.0/lastMonthPrice
|
||||
// if(Math.abs(markup)>10){
|
||||
// def random = 0
|
||||
// if(dbo.get("avg_price")-lastMonthPrice>=0){
|
||||
// random = com.fangjia.core.util.RandomUtil.random(1, 9)
|
||||
// }else{
|
||||
// random = com.fangjia.core.util.RandomUtil.random(-9, 0)
|
||||
// }
|
||||
// def newPrice = lastMonthPrice*random/100 + lastMonthPrice
|
||||
// m << newPrice.toInteger()
|
||||
// }else{
|
||||
// m << dbo.get("avg_price")
|
||||
// }
|
||||
// }else{
|
||||
// m << dbo.get("avg_price")
|
||||
// }
|
||||
m << curDate.toInteger();
|
||||
result << m
|
||||
}
|
||||
|
||||
@@ -48,51 +48,51 @@ import org.bson.types.ObjectId
|
||||
// return location;
|
||||
//}
|
||||
|
||||
//List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202109.csv"))
|
||||
//def map1 = [:]
|
||||
//for(def s : list1){
|
||||
// def row = s.split("@\\|@")
|
||||
// def key = row[2]+"##"+row[3]+"##"+row[4]
|
||||
// map1.put(key, row[5])
|
||||
//}
|
||||
//
|
||||
//List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202108.csv"))
|
||||
//def map2 = [:]
|
||||
//for(def s : list2) {
|
||||
// def row = s.split("@\\|@")
|
||||
// def key = row[2] + "##" + row[3] + "##" + row[4]
|
||||
// map2.put(key, row[5])
|
||||
//}
|
||||
//def rows = []
|
||||
//map1.each {k,v->
|
||||
// def row = []
|
||||
// def keys = k.split("##")
|
||||
// def lastPrice = map2.get(k)
|
||||
// rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString()]
|
||||
//}
|
||||
//ExcelUtils.write("D:\\数据\\福州\\价格比对.xlsx", rows, "数据")
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州涨幅在10%以上小区.xlsx",{d->
|
||||
def obj = CommonDatautils.queryDistrictPrice(d['A'],d['B'], d['C'], DateUtils.parseFromTightByPattern("2021-10-10","yyyy-MM-dd"))
|
||||
if(obj){
|
||||
if(obj.value_type == 1){
|
||||
d["G"] = "指导价"
|
||||
}else if(obj.value_type == 2){
|
||||
d["G"] ="自身样本"
|
||||
}else if(obj.value_type == 3){
|
||||
d["G"] = "第三方"
|
||||
}else if(obj.value_type == 4){
|
||||
d["G"] = "交易数据"
|
||||
}else if(obj.value_type == 5){
|
||||
d["G"] ="周边楼盘"
|
||||
}else if(obj.value_type == 6){
|
||||
d["G"] ="推导"
|
||||
}else if(obj.value_type == 9){
|
||||
d["G"] = "神经网络"
|
||||
}else if(obj.value_type == 10){
|
||||
d["G"] = "无方式"
|
||||
}
|
||||
}
|
||||
},0)
|
||||
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202112.csv"))
|
||||
def map1 = [:]
|
||||
for(def s : list1){
|
||||
def row = s.split("@\\|@")
|
||||
def key = row[2]+"##"+row[3]+"##"+row[4]
|
||||
map1.put(key, row[5])
|
||||
}
|
||||
|
||||
List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202111.csv"))
|
||||
def map2 = [:]
|
||||
for(def s : list2) {
|
||||
def row = s.split("@\\|@")
|
||||
def key = row[2] + "##" + row[3] + "##" + row[4]
|
||||
map2.put(key, row[5])
|
||||
}
|
||||
def rows = []
|
||||
map1.each {k,v->
|
||||
def row = []
|
||||
def keys = k.split("##")
|
||||
def lastPrice = map2.get(k)
|
||||
rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString()]
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\福州\\价格比对12-2.xlsx", rows, "数据")
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州涨幅在10%以上小区.xlsx",{d->
|
||||
// def obj = CommonDatautils.queryDistrictPrice(d['A'],d['B'], d['C'], DateUtils.parseFromTightByPattern("2021-10-10","yyyy-MM-dd"))
|
||||
// if(obj){
|
||||
// if(obj.value_type == 1){
|
||||
// d["G"] = "指导价"
|
||||
// }else if(obj.value_type == 2){
|
||||
// d["G"] ="自身样本"
|
||||
// }else if(obj.value_type == 3){
|
||||
// d["G"] = "第三方"
|
||||
// }else if(obj.value_type == 4){
|
||||
// d["G"] = "交易数据"
|
||||
// }else if(obj.value_type == 5){
|
||||
// d["G"] ="周边楼盘"
|
||||
// }else if(obj.value_type == 6){
|
||||
// d["G"] ="推导"
|
||||
// }else if(obj.value_type == 9){
|
||||
// d["G"] = "神经网络"
|
||||
// }else if(obj.value_type == 10){
|
||||
// d["G"] = "无方式"
|
||||
// }
|
||||
// }
|
||||
//},0)
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
||||
//福州第三方均价统计,发送钉钉消息
|
||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202111.csv")
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202112.csv")
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
|
||||
+26
-16
@@ -62,7 +62,8 @@ class DistrictIndexBatch extends BaseBatch{
|
||||
|
||||
def run(){
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def citys = ["上海"]//CommonDatautils.findAllCity();
|
||||
def citys = CommonDatautils.findAllCity();
|
||||
citys.remove("上海")
|
||||
for(def city : citys){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2"])
|
||||
// dists = dists[0..20]
|
||||
@@ -156,13 +157,19 @@ class DistrictIndexBatch extends BaseBatch{
|
||||
fileName = "D:\\数据\\评级\\上海综合潜力.dat"
|
||||
}
|
||||
|
||||
FileUtils.batchCreateFile(datas, fileName, headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile(datas, fileName, headers, "utf-8", true);
|
||||
// ExcelUtils.write("D:\\数据\\评级\\上海综合潜力.xlsx", datas, "数据")
|
||||
}
|
||||
|
||||
def getHousePrice(d, dists, priceMap, citycode){
|
||||
//def areas = findHouse(d, 20200101, 20220101)
|
||||
def areas = findHouseByOracle(d._id.toString(), citycode);
|
||||
|
||||
def areas = []
|
||||
if(citycode){
|
||||
areas = findHouseByOracle(d._id.toString(), citycode);
|
||||
}else{
|
||||
areas = findHouse(d, 20200101, 20220101)
|
||||
}
|
||||
def avg_area = null
|
||||
if(areas){
|
||||
avg_area = areas.sum()/areas.size();
|
||||
@@ -188,24 +195,27 @@ class DistrictIndexBatch extends BaseBatch{
|
||||
|
||||
if(selectDists){
|
||||
println d.region+" "+d.name+" 周边同建筑年代楼盘:"+selectDists.size()
|
||||
// def allAreas = []
|
||||
def allAreas = []
|
||||
def ids = []
|
||||
for(def sd : selectDists){
|
||||
// //def areaDatas = findHouse(sd, 20190101, 20220101
|
||||
// def areaDatas = findHouseByOracle(sd._id.toString());
|
||||
// allAreas.addAll(areaDatas)
|
||||
ids << sd._id.toString()
|
||||
}
|
||||
if(ids.size()>=901){
|
||||
ids = ids[0..900]
|
||||
}
|
||||
if(citycode){
|
||||
for(def sd : selectDists){
|
||||
ids << sd._id.toString()
|
||||
}
|
||||
if(ids.size()>=901){
|
||||
ids = ids[0..900]
|
||||
}
|
||||
|
||||
def idstr = ids.join("','")
|
||||
def allAreas = findHouseByOracle2(idstr,citycode);
|
||||
def idstr = ids.join("','")
|
||||
allAreas = findHouseByOracle2(idstr,citycode);
|
||||
}else{
|
||||
for(def sd : selectDists){
|
||||
def areaDatas = findHouse(sd, 20190101, 20220101)
|
||||
allAreas.addAll(areaDatas)
|
||||
}
|
||||
}
|
||||
if(allAreas){
|
||||
avg_area = allAreas.sum()/allAreas.size();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+7
-3
@@ -98,9 +98,9 @@ class CrawalUtil {
|
||||
}
|
||||
|
||||
|
||||
static def getDocumentByABuYun(def url, def errorMsg, def charset, def language = null) {
|
||||
static def getDocumentByABuYun(def url, def errorMsg, def charset, def language = null, def isJson=false) {
|
||||
def flag = true;
|
||||
Document document = null;
|
||||
def document = null;
|
||||
def count = 0;
|
||||
while (flag) {
|
||||
def content = null;
|
||||
@@ -121,7 +121,11 @@ class CrawalUtil {
|
||||
}
|
||||
}
|
||||
if (content) {
|
||||
document = Jsoup.parse(content);
|
||||
if(!isJson){
|
||||
document = Jsoup.parse(content);
|
||||
}else{
|
||||
document = content
|
||||
}
|
||||
}
|
||||
if (!content || (content && errorMsg && content.contains(errorMsg))) {
|
||||
count++;
|
||||
|
||||
+1
-1
@@ -364,7 +364,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
def standardDistrict(def data, def avg_price){
|
||||
//名称匹配,名称别名匹配,地址别名匹配
|
||||
//def district = getMatchName(data.city, data.region, data.name, data.address,"district")
|
||||
data.name = data.name.replace("(","").replace(")","")
|
||||
data.name = data.name.replace("(","").replace(")","").replace("数据由万科物业提供","").trim()
|
||||
def district = null;
|
||||
def type = 0;
|
||||
if(data.name.contains("写字楼")||data.name.contains("办公楼")||data.name.contains("商业")){
|
||||
|
||||
+4
-4
@@ -40,11 +40,11 @@ class DataStat {
|
||||
|
||||
|
||||
def run(){
|
||||
def source = "城市房产"//
|
||||
def citys = findCrawlCity(source, 202110);
|
||||
def source = "58同城"//
|
||||
def citys = findCrawlCity(source, 202111);
|
||||
for(def city : citys){
|
||||
def count1 = findCount(source, city, 202110);
|
||||
def count2 = findCount(source, city, 202111);
|
||||
def count1 = findCount(source, city, 202111);
|
||||
def count2 = findCount(source, city, 202112);
|
||||
println city +" "+count1+" "+count2
|
||||
// println count1
|
||||
// println "--- ----"
|
||||
|
||||
+55
-18
File diff suppressed because one or more lines are too long
+2
-2
@@ -17,8 +17,8 @@ class SynchDistrictPrice {
|
||||
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
|
||||
// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
|
||||
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
|
||||
def sourceMap = ["58.com":"58同城","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多",
|
||||
"anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
|
||||
//"fang.com":"搜房",
|
||||
def sourceMap = ["fang.com":"搜房"]
|
||||
static main(args) {
|
||||
def task = new SynchDistrictPrice()
|
||||
task.run()
|
||||
|
||||
Reference in New Issue
Block a user