This commit is contained in:
2022-01-14 10:54:04 +08:00
parent 0ff7f23f1e
commit 40928486ea
17 changed files with 513 additions and 170 deletions
@@ -82,9 +82,10 @@ class CcbCommonUtils {
return rets
}
public static def findGuidePrice(city){
public static def findGuidePrice(city, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put('c_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt', DateUtils.parseFromTightYYYYMMDD(endDate)));
def dbcursor = DISTRICT_GUIDE_PRICE_COLL.find(q)
def rets = []
for(def d : dbcursor){
@@ -25,7 +25,7 @@ class DistrictGuidePriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findGuidePrice(city)
def prices = CcbCommonUtils.findGuidePrice(city, "2022-01-1", "2022-02-1")
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
@@ -24,12 +24,14 @@ class ExportOracleData {
}
def run() {
//
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
// exportGuidedPrice()
// exportCommunityGuidedPrice()
exportCommunityGuidedPrice(lastMonth.toString()+"28")
//
// //exportThirdPartyPrice()//废弃掉,老表
// exportThirdPartyPrice2(202111)
// exportThirdPartyPrice2(lastMonth)
//
//
//
@@ -40,20 +42,20 @@ class ExportOracleData {
// exportCommunityStaging()
// exportCommunityBuileType()
// exportBigdataDistrictData()
exportBigdataCommunity()
// exportBigdataCommunity()
// exportBigdataDeal("2021-11-01")
// exportAvmEvaluationGJNB(202112)
// exportAVMEvaluationItemGJNB(202112)
// exportCommunityBasePrice(202112)
// exportAvmEvaluationGJNB(curMonth)
// exportAVMEvaluationItemGJNB(curMonth)
// exportCommunityBasePrice(curMonth)
//
// exportCityPrice(202112)
// exportRegionPrice(202112)
// exportCityPrice(curMonth)
// exportRegionPrice(curMonth)
// exportCommunityBasePriceGj(202112)
// exportCommunityBasePriceGj(curMonth)
// exportCommunityPdescribeGj(202112)
// exportCommCaseDetailGj(202112)
// exportCommunityPdescribeGj(curMonth)
// exportCommCaseDetailGj(curMonth)
// exportLastFile()
}
@@ -255,26 +257,26 @@ class ExportOracleData {
println "GUIDED_PRICE_GJ 上传Ftp完成......"
}
def exportCommunityGuidedPrice() {
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj";
def exportCommunityGuidedPrice(def date) {
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where createtime>=to_date('${date}','yyyyMMdd')";
def allRows = []
// sqlInstance.eachRow(sql, { r ->
// def data = []
// data << r.ID
// data << r.COMMID
// data << r.COMMNAME
// data << r.CITYID
// data << r.CITYNAME
// data << r.DISTID
// data << r.DISTNAME
// data << r.AVGPRICE
// data << r.SOURCETYPE
// data << r.REMARK
// data << r.CREATETIME
// data << r.UPDATETIME
// data << r.SOURCECODE
// allRows << data
// });
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.COMMID
data << r.COMMNAME
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.AVGPRICE
data << r.SOURCETYPE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2021-10-1", "2021-11-1")
def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-1", "2022-02-1")
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
@@ -4,6 +4,8 @@ import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.fangjia.core.RegionData
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
class LoadGuide {
public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017)
@@ -14,13 +16,31 @@ class LoadGuide {
}
def run(){
// match("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
// saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞指导价218.xlsx")
// saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞二手房政府指导价.xls")
// saveChengduoPrice("D:\\数据\\建行\\房产估值\\指导价\\成都部分住宅小区二手房成交参考价格201-286-258.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\绍兴指导价小区_0909.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\宁波交易参考价格.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安成交参考价格.xlsx")
savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\无锡100 广州96 二手住房成交参考价格表.xlsx")
savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx")
}
def match(def filename){
def city = "西安"
ExcelUtils.modify(filename,{d->
def newRegion = RegionData.formatRegion(city, d['A'])
if(newRegion){
def rMap = SeaweedDataMatchProcess.doMatch(city, newRegion, d['B'], null);
if(rMap){
d['D'] = city
d['E'] = rMap.region
d['F'] = rMap.name
}
}
},0)
}
@@ -32,8 +52,8 @@ class LoadGuide {
def o = new BasicDBObject()
o.city = d['E']
o.region = d['A']
o.name = d['C']
o.avg_price = d['D'].toString().toDouble().toInteger()
o.name = d['B']
o.avg_price = d['C'].toString().toDouble().toInteger()
o.match_city = d['E']
o.match_region = d['F']
o.match_name = d['G']
@@ -57,11 +77,11 @@ class LoadGuide {
o.city = d['E']
o.region = d['A']
o.name = d['B']
o.avg_price = d['D'].toString().toDouble().toInteger()
o.avg_price = d['C'].toString().toDouble().toInteger()
o.match_city = d['E']
o.match_region = d['F']
o.match_name = d['G']
o.house_count = d['C'].toString().toDouble().toInteger()
// o.house_count = d['C'].toString().toDouble().toInteger()
o.source_type = 1 //1:成交参考价 2:参考价
o.c_date = new Date()
rets << o
@@ -4,7 +4,107 @@ import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d ->
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
// if(newRegion && !d['K']) {
// def rMap = SeaweedDataMatchProcess.doMatch(d['A'], newRegion, d['C'], d['D']+d['E']);
// if(!rMap && d['G']){
// rMap = SeaweedDataMatchProcess.doMatch(d['A'], newRegion, d['G'], d['H']);
// }
// if(rMap){
// d['O'] = rMap.city
// d['P'] = rMap.region
// d['Q'] = rMap.name
// d['R'] = rMap.address
// }
// }
//});
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\高德反查数据3_2003.xlsx",{d->
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
// if(newRegion && !d['K']){
//
// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'],"district")
// if(dist){
// d['K']="完全一致"
// d['L'] = dist.estate_type2
// d['M'] = dist.name_cat
// }else{
// dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['C'],"district")
// if(dist){
// d['K']="映射别名"
// d['L'] = dist.estate_type2
// d['M'] = dist.name_cat
// }else{
// def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['C'],"district")
// if(rets){
// dist = rets[0]
// }
// if(rets){
// d['K']="别名"
// d['L'] = dist.estate_type2
// d['M'] = dist.name_cat
// }
// }
// }
// }
//
//},0)
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\高德反查数据3_2003.xlsx",{d->
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
// if(newRegion && !d['K'] && d['G']){
//
// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['G'],"district")
// if(dist){
// d['K']="完全一致(高德)"
// d['L'] = dist.estate_type2
// d['M'] = dist.name_cat
// }else{
// dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['G'],"district")
// if(dist){
// d['K']="映射别名(高德)"
// d['L'] = dist.estate_type2
// d['M'] = dist.name_cat
// }else{
// def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['G'],"district")
// if(rets){
// dist = rets[0]
// }
// if(rets){
// d['K']="别名(高德)"
// d['L'] = dist.estate_type2
// d['M'] = dist.name_cat
// }
// }
// }
// }
//
//},0)
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\价格过滤小区融合情况_1209.xlsx",{d->
// def dist = CommonDatautils.findDistrictFieldById(d['F'], ["address","lat","lng"])
// if(dist){
// d['Z'] = dist.address
// if(dist.lat){
// d['AA'] = dist.lat/10000000000.0
// d['AB'] = dist.lng/10000000000.0
// }
// }
//},0)
//def cityDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\小区\\重庆高德反查数据_1529.xlsx",0,1)
//def cityDataMap = [:]
@@ -74,54 +174,156 @@ def city = "
//},0)
//ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例.xlsx",{d->
//ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例(合并).xlsx",{d->
// def obj = CommonDatautils.findDistrict("广州", d['C'], d['D'], "district")
// if(obj){
// def year = CommonUtils.getDistrictAge(obj)
// if(year){
// d['H'] = year
// }
// if(obj.alias){
// d['K'] = obj.alias.join(" ")
//// def year = CommonUtils.getDistrictAge(obj)
//// if(year){
//// d['H'] = year
//// }
//// if(obj.alias){
//// d['K'] = obj.alias.join(" ")
//// }
// if(obj.mapping_alias){
// d['X'] = obj.mapping_alias.join(" ")
// }
//
// }
//},0)
def data1s = ExcelUtils.read("D:\\数据\\广发\\图片转换汇总1.8.xlsx",0, 1)
def scoreMap = [:]
for(def d : data1s){
scoreMap.put(d['B'], d)
}
//def data1s = ExcelUtils.read("D:\\数据\\广发\\图片转换汇总1.8.xlsx",0, 1)
//def scoreMap = [:]
//for(def d : data1s){
// scoreMap.put(d['B'], d)
//}
//
//def data2s = ExcelUtils.read("D:\\数据\\广发\\图片转换汇总1.8.xlsx",1, 1)
//def levelMap = [:]
//for(def d : data2s){
// levelMap.put(d['E'], d)
//}
//
//ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例.xlsx",{d->
// if(d['E']){
// def row = scoreMap.get(d['E'])
// if(row){
// d['I'] = row['G']
// d['J'] = row['H']
// d['K'] = row['I']
// d['L'] = row['J']
// d['M'] = row['K']
// d['N'] = row['L']
// d['O'] = row['M']
// }
//
// def data = levelMap.get(d['E'])
// if(data){
// d['P'] = data['G']
// d['Q'] = data['H']
// d['R'] = data['I']
// d['S'] = data['J']
// d['T'] = data['K']
// d['U'] = data['L']
// d['V'] = data['M']
// }
// }
//},0)
def data2s = ExcelUtils.read("D:\\数据\\广发\\图片转换汇总1.8.xlsx",1, 1)
def levelMap = [:]
for(def d : data2s){
levelMap.put(d['E'], d)
}
//ExcelUtils.modify("D:\\数据\\成交\\福州成交.xlsx",{d->
// def o = new BasicDBObject()
// o.area = d['D'].toDouble()
// o.avg_price = d['F'].toInteger()
// o.city = "福州"
// o.district_name = d['C']
// o.region = d['B']
// o.source = "贝壳找房"
// o.trade_date = DateUtils.parseFromTightYYYYMMDD(d['M'])
// o.total_price = d['E'].toInteger()*10000
// if(d['I']){
// o.hall = d['I'].toInteger()
// }
//
// o.m_date = new Date()
// if(d['H']){
// o.room = d['H'].toInteger()
// }
//
// o.create_date = new Date()
// o.status = 2
// DEAL_PRICE.save(o)
//
//},0)
ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例.xlsx",{d->
if(d['E']){
def row = scoreMap.get(d['E'])
if(row){
d['I'] = row['G']
d['J'] = row['H']
d['K'] = row['I']
d['L'] = row['J']
d['M'] = row['K']
d['N'] = row['L']
d['O'] = row['M']
}
def data = levelMap.get(d['E'])
if(data){
d['P'] = data['G']
d['Q'] = data['H']
d['R'] = data['I']
d['S'] = data['J']
d['T'] = data['K']
d['U'] = data['L']
d['V'] = data['M']
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d ->
// if(d['K'] && d['K']=="别名"){
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
// if(newRegion){
// def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['C'],"district")
// def dist = null
// if(rets){
// dist = rets[0]
// }
// if(dist){
// def alias = dist.mapping_alias
// if(!alias){
// alias = []
// }
// if(!alias.contains(d['C'])){
// alias << d['C']
// }
// def udata = new BasicDBObject()
// udata.put("mapping_alias",alias)
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
// }
// }
// }
//});
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d ->
// if(d['K'] && d['K']=="映射别名(高德)" && d['G']){
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
// if(newRegion){
// dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['G'],"district")
// if(dist){
// def alias = dist.mapping_alias
// if(!alias){
// alias = []
// }
// if(!alias.contains(d['C'])){
// alias << d['C']
// }
// def udata = new BasicDBObject()
// udata.put("mapping_alias",alias)
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
// }
// }
// }
//});
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d ->
if(d['K'] && d['K']=="别名(高德)" && d['G']){
def newRegion = RegionData.formatRegion(d['A'], d['B'])
if(newRegion){
def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['G'],"district")
def dist = null
if(rets){
dist = rets[0]
}
if(dist){
def alias = dist.mapping_alias
if(!alias){
alias = []
}
if(!alias.contains(d['C'])){
alias << d['C']
}
def udata = new BasicDBObject()
udata.put("mapping_alias",alias)
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
}
}
}
},0)
});
@@ -27,6 +27,7 @@ class ExportDistrictRongHe extends BaseBatch{
def citys = CcbCommonUtils.citys;
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
//检查ok文件是否上传
// exportAllBigdataDealFJW()
for(def city : citys) {
def cityCode = cityCodeDataMap.get(city)
@@ -47,6 +48,7 @@ class ExportDistrictRongHe extends BaseBatch{
exportBigdataCommlayoutpicFjw(cityCode)
}
exportEvaluateRate()
exportOk()
}
@@ -593,4 +595,14 @@ class ExportDistrictRongHe extends BaseBatch{
println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......"
}
def exportOk(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok"
if(CommonDatautils.isWindow()){
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok"
}
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload(dataCodeFileName, curDate)
}
}
@@ -13,7 +13,34 @@ class DownPdfCheck {
static main(args) {
def task = new DownPdfCheck();
task.run();
// task.run();
task.czLogRun()
}
def czLogRun(){
def files = getAllFiles("D:\\数据\\常州")
if(files){
for(def filepath : files){
read(filepath);
}
}
}
def read(file){
// File file = new File(fileName);
def index = 0
file.eachLine{it->
if(it.contains("4ead0347b4055be6989e726e")||it.contains("61a2e0")){
index++
println it
}
}
}
def run(){
@@ -8,6 +8,7 @@ import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.jusfoun.JusfounCommon
import com.fangjia.script.util.DateUtils
import org.apache.commons.codec.digest.DigestUtils
import org.apache.commons.io.FileUtils
import java.util.concurrent.TimeUnit
@@ -79,6 +80,14 @@ class DistrictPriceExportBatch extends BaseBatch{
def lastDate = new Date()
def startDate = DateUtils.getFirstDayOfMonth(lastDate, -1)
// List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202111.csv"))
// def map2 = [:]
// for(def s : list2) {
// def row = s.split("@\\|@")
// def key = row[2] + "##" + row[3] + "##" + row[4]
// map2.put(key, row[5])
// }
def result = []
for(def rMap : districts){
def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate)
@@ -90,6 +99,25 @@ class DistrictPriceExportBatch extends BaseBatch{
m << rMap.region
m << rMap.name
m << dbo.get("avg_price")
// def lastMonthPrice = map2.get(rMap.city + "##" + rMap.region+"##"+rMap.name)
// if(lastMonthPrice && dbo.get("avg_price")){
// lastMonthPrice = lastMonthPrice.toInteger()
// def markup = (dbo.get("avg_price")-lastMonthPrice)*100.0/lastMonthPrice
// if(Math.abs(markup)>10){
// def random = 0
// if(dbo.get("avg_price")-lastMonthPrice>=0){
// random = com.fangjia.core.util.RandomUtil.random(1, 9)
// }else{
// random = com.fangjia.core.util.RandomUtil.random(-9, 0)
// }
// def newPrice = lastMonthPrice*random/100 + lastMonthPrice
// m << newPrice.toInteger()
// }else{
// m << dbo.get("avg_price")
// }
// }else{
// m << dbo.get("avg_price")
// }
m << curDate.toInteger();
result << m
}
@@ -48,51 +48,51 @@ import org.bson.types.ObjectId
// return location;
//}
//List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202109.csv"))
//def map1 = [:]
//for(def s : list1){
// def row = s.split("@\\|@")
// def key = row[2]+"##"+row[3]+"##"+row[4]
// map1.put(key, row[5])
//}
//
//List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202108.csv"))
//def map2 = [:]
//for(def s : list2) {
// def row = s.split("@\\|@")
// def key = row[2] + "##" + row[3] + "##" + row[4]
// map2.put(key, row[5])
//}
//def rows = []
//map1.each {k,v->
// def row = []
// def keys = k.split("##")
// def lastPrice = map2.get(k)
// rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString()]
//}
//ExcelUtils.write("D:\\数据\\福州\\价格比对.xlsx", rows, "数据")
ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州涨幅在10%以上小区.xlsx",{d->
def obj = CommonDatautils.queryDistrictPrice(d['A'],d['B'], d['C'], DateUtils.parseFromTightByPattern("2021-10-10","yyyy-MM-dd"))
if(obj){
if(obj.value_type == 1){
d["G"] = "指导价"
}else if(obj.value_type == 2){
d["G"] ="自身样本"
}else if(obj.value_type == 3){
d["G"] = "第三方"
}else if(obj.value_type == 4){
d["G"] = "交易数据"
}else if(obj.value_type == 5){
d["G"] ="周边楼盘"
}else if(obj.value_type == 6){
d["G"] ="推导"
}else if(obj.value_type == 9){
d["G"] = "神经网络"
}else if(obj.value_type == 10){
d["G"] = "无方式"
}
}
},0)
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202112.csv"))
def map1 = [:]
for(def s : list1){
def row = s.split("@\\|@")
def key = row[2]+"##"+row[3]+"##"+row[4]
map1.put(key, row[5])
}
List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202111.csv"))
def map2 = [:]
for(def s : list2) {
def row = s.split("@\\|@")
def key = row[2] + "##" + row[3] + "##" + row[4]
map2.put(key, row[5])
}
def rows = []
map1.each {k,v->
def row = []
def keys = k.split("##")
def lastPrice = map2.get(k)
rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString()]
}
ExcelUtils.write("D:\\数据\\福州\\价格比对12-2.xlsx", rows, "数据")
//ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州涨幅在10%以上小区.xlsx",{d->
// def obj = CommonDatautils.queryDistrictPrice(d['A'],d['B'], d['C'], DateUtils.parseFromTightByPattern("2021-10-10","yyyy-MM-dd"))
// if(obj){
// if(obj.value_type == 1){
// d["G"] = "指导价"
// }else if(obj.value_type == 2){
// d["G"] ="自身样本"
// }else if(obj.value_type == 3){
// d["G"] = "第三方"
// }else if(obj.value_type == 4){
// d["G"] = "交易数据"
// }else if(obj.value_type == 5){
// d["G"] ="周边楼盘"
// }else if(obj.value_type == 6){
// d["G"] ="推导"
// }else if(obj.value_type == 9){
// d["G"] = "神经网络"
// }else if(obj.value_type == 10){
// d["G"] = "无方式"
// }
// }
//},0)
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202111.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202112.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -62,7 +62,8 @@ class DistrictIndexBatch extends BaseBatch{
def run(){
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def citys = ["上海"]//CommonDatautils.findAllCity();
def citys = CommonDatautils.findAllCity();
citys.remove("上海")
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2"])
// dists = dists[0..20]
@@ -156,13 +157,19 @@ class DistrictIndexBatch extends BaseBatch{
fileName = "D:\\数据\\评级\\上海综合潜力.dat"
}
FileUtils.batchCreateFile(datas, fileName, headers, "utf-8", false);
FileUtils.batchCreateFile(datas, fileName, headers, "utf-8", true);
// ExcelUtils.write("D:\\数据\\评级\\上海综合潜力.xlsx", datas, "数据")
}
def getHousePrice(d, dists, priceMap, citycode){
//def areas = findHouse(d, 20200101, 20220101)
def areas = findHouseByOracle(d._id.toString(), citycode);
def areas = []
if(citycode){
areas = findHouseByOracle(d._id.toString(), citycode);
}else{
areas = findHouse(d, 20200101, 20220101)
}
def avg_area = null
if(areas){
avg_area = areas.sum()/areas.size();
@@ -188,24 +195,27 @@ class DistrictIndexBatch extends BaseBatch{
if(selectDists){
println d.region+" "+d.name+" 周边同建筑年代楼盘:"+selectDists.size()
// def allAreas = []
def allAreas = []
def ids = []
for(def sd : selectDists){
// //def areaDatas = findHouse(sd, 20190101, 20220101
// def areaDatas = findHouseByOracle(sd._id.toString());
// allAreas.addAll(areaDatas)
ids << sd._id.toString()
}
if(ids.size()>=901){
ids = ids[0..900]
}
if(citycode){
for(def sd : selectDists){
ids << sd._id.toString()
}
if(ids.size()>=901){
ids = ids[0..900]
}
def idstr = ids.join("','")
def allAreas = findHouseByOracle2(idstr,citycode);
def idstr = ids.join("','")
allAreas = findHouseByOracle2(idstr,citycode);
}else{
for(def sd : selectDists){
def areaDatas = findHouse(sd, 20190101, 20220101)
allAreas.addAll(areaDatas)
}
}
if(allAreas){
avg_area = allAreas.sum()/allAreas.size();
}
}
}
}
@@ -98,9 +98,9 @@ class CrawalUtil {
}
static def getDocumentByABuYun(def url, def errorMsg, def charset, def language = null) {
static def getDocumentByABuYun(def url, def errorMsg, def charset, def language = null, def isJson=false) {
def flag = true;
Document document = null;
def document = null;
def count = 0;
while (flag) {
def content = null;
@@ -121,7 +121,11 @@ class CrawalUtil {
}
}
if (content) {
document = Jsoup.parse(content);
if(!isJson){
document = Jsoup.parse(content);
}else{
document = content
}
}
if (!content || (content && errorMsg && content.contains(errorMsg))) {
count++;
@@ -364,7 +364,7 @@ class DataStandardBatch extends BaseBatch{
def standardDistrict(def data, def avg_price){
//名称匹配,名称别名匹配,地址别名匹配
//def district = getMatchName(data.city, data.region, data.name, data.address,"district")
data.name = data.name.replace("(","").replace(")","")
data.name = data.name.replace("(","").replace(")","").replace("数据由万科物业提供","").trim()
def district = null;
def type = 0;
if(data.name.contains("写字楼")||data.name.contains("办公楼")||data.name.contains("商业")){
@@ -40,11 +40,11 @@ class DataStat {
def run(){
def source = "城市房产"//
def citys = findCrawlCity(source, 202110);
def source = "58同城"//
def citys = findCrawlCity(source, 202111);
for(def city : citys){
def count1 = findCount(source, city, 202110);
def count2 = findCount(source, city, 202111);
def count1 = findCount(source, city, 202111);
def count2 = findCount(source, city, 202112);
println city +" "+count1+" "+count2
// println count1
// println "--- ----"
File diff suppressed because one or more lines are too long
@@ -17,8 +17,8 @@ class SynchDistrictPrice {
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
def sourceMap = ["58.com":"58同城","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多",
"anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
//"fang.com":"搜房",
def sourceMap = ["fang.com":"搜房"]
static main(args) {
def task = new SynchDistrictPrice()
task.run()