update
This commit is contained in:
+2
-1
@@ -82,9 +82,10 @@ class CcbCommonUtils {
|
|||||||
return rets
|
return rets
|
||||||
}
|
}
|
||||||
|
|
||||||
public static def findGuidePrice(city){
|
public static def findGuidePrice(city, startDate, endDate){
|
||||||
def q = new BasicDBObject()
|
def q = new BasicDBObject()
|
||||||
q.put("city", city)
|
q.put("city", city)
|
||||||
|
q.put('c_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt', DateUtils.parseFromTightYYYYMMDD(endDate)));
|
||||||
def dbcursor = DISTRICT_GUIDE_PRICE_COLL.find(q)
|
def dbcursor = DISTRICT_GUIDE_PRICE_COLL.find(q)
|
||||||
def rets = []
|
def rets = []
|
||||||
for(def d : dbcursor){
|
for(def d : dbcursor){
|
||||||
|
|||||||
+1
-1
@@ -25,7 +25,7 @@ class DistrictGuidePriceBatch extends BaseBatch{
|
|||||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||||
|
|
||||||
//添加自己库中的人工指导价
|
//添加自己库中的人工指导价
|
||||||
def prices = CcbCommonUtils.findGuidePrice(city)
|
def prices = CcbCommonUtils.findGuidePrice(city, "2022-01-1", "2022-02-1")
|
||||||
if(prices){
|
if(prices){
|
||||||
def distMap = getDistrictMap(city)
|
def distMap = getDistrictMap(city)
|
||||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||||
|
|||||||
+33
-31
@@ -24,12 +24,14 @@ class ExportOracleData {
|
|||||||
}
|
}
|
||||||
|
|
||||||
def run() {
|
def run() {
|
||||||
//
|
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
|
||||||
|
|
||||||
|
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||||
// exportGuidedPrice()
|
// exportGuidedPrice()
|
||||||
// exportCommunityGuidedPrice()
|
exportCommunityGuidedPrice(lastMonth.toString()+"28")
|
||||||
//
|
//
|
||||||
// //exportThirdPartyPrice()//废弃掉,老表
|
// //exportThirdPartyPrice()//废弃掉,老表
|
||||||
// exportThirdPartyPrice2(202111)
|
// exportThirdPartyPrice2(lastMonth)
|
||||||
//
|
//
|
||||||
//
|
//
|
||||||
//
|
//
|
||||||
@@ -40,20 +42,20 @@ class ExportOracleData {
|
|||||||
// exportCommunityStaging()
|
// exportCommunityStaging()
|
||||||
// exportCommunityBuileType()
|
// exportCommunityBuileType()
|
||||||
// exportBigdataDistrictData()
|
// exportBigdataDistrictData()
|
||||||
exportBigdataCommunity()
|
// exportBigdataCommunity()
|
||||||
// exportBigdataDeal("2021-11-01")
|
// exportBigdataDeal("2021-11-01")
|
||||||
|
|
||||||
// exportAvmEvaluationGJNB(202112)
|
// exportAvmEvaluationGJNB(curMonth)
|
||||||
// exportAVMEvaluationItemGJNB(202112)
|
// exportAVMEvaluationItemGJNB(curMonth)
|
||||||
// exportCommunityBasePrice(202112)
|
// exportCommunityBasePrice(curMonth)
|
||||||
//
|
//
|
||||||
// exportCityPrice(202112)
|
// exportCityPrice(curMonth)
|
||||||
// exportRegionPrice(202112)
|
// exportRegionPrice(curMonth)
|
||||||
|
|
||||||
// exportCommunityBasePriceGj(202112)
|
// exportCommunityBasePriceGj(curMonth)
|
||||||
|
|
||||||
// exportCommunityPdescribeGj(202112)
|
// exportCommunityPdescribeGj(curMonth)
|
||||||
// exportCommCaseDetailGj(202112)
|
// exportCommCaseDetailGj(curMonth)
|
||||||
// exportLastFile()
|
// exportLastFile()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -255,26 +257,26 @@ class ExportOracleData {
|
|||||||
println "GUIDED_PRICE_GJ 上传Ftp完成......"
|
println "GUIDED_PRICE_GJ 上传Ftp完成......"
|
||||||
}
|
}
|
||||||
|
|
||||||
def exportCommunityGuidedPrice() {
|
def exportCommunityGuidedPrice(def date) {
|
||||||
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj";
|
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where createtime>=to_date('${date}','yyyyMMdd')";
|
||||||
def allRows = []
|
def allRows = []
|
||||||
// sqlInstance.eachRow(sql, { r ->
|
sqlInstance.eachRow(sql, { r ->
|
||||||
// def data = []
|
def data = []
|
||||||
// data << r.ID
|
data << r.ID
|
||||||
// data << r.COMMID
|
data << r.COMMID
|
||||||
// data << r.COMMNAME
|
data << r.COMMNAME
|
||||||
// data << r.CITYID
|
data << r.CITYID
|
||||||
// data << r.CITYNAME
|
data << r.CITYNAME
|
||||||
// data << r.DISTID
|
data << r.DISTID
|
||||||
// data << r.DISTNAME
|
data << r.DISTNAME
|
||||||
// data << r.AVGPRICE
|
data << r.AVGPRICE
|
||||||
// data << r.SOURCETYPE
|
data << r.SOURCETYPE
|
||||||
// data << r.REMARK
|
data << r.REMARK
|
||||||
// data << r.CREATETIME
|
data << r.CREATETIME
|
||||||
// data << r.UPDATETIME
|
data << r.UPDATETIME
|
||||||
// data << r.SOURCECODE
|
data << r.SOURCECODE
|
||||||
// allRows << data
|
allRows << data
|
||||||
// });
|
});
|
||||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
|
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
|
||||||
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||||
if (CommonDatautils.isWindow()) {
|
if (CommonDatautils.isWindow()) {
|
||||||
|
|||||||
+1
-1
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
|||||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||||
|
|
||||||
//添加自己库中的人工指导价
|
//添加自己库中的人工指导价
|
||||||
def prices = CcbCommonUtils.findPersonPrice(city, "2021-10-1", "2021-11-1")
|
def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-1", "2022-02-1")
|
||||||
if(prices){
|
if(prices){
|
||||||
def distMap = getDistrictMap(city)
|
def distMap = getDistrictMap(city)
|
||||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||||
|
|||||||
+26
-6
@@ -4,6 +4,8 @@ import com.fangjia.script.util.ExcelUtils
|
|||||||
import com.fangjia.script.util.MongoUtils
|
import com.fangjia.script.util.MongoUtils
|
||||||
import com.mongodb.BasicDBObject
|
import com.mongodb.BasicDBObject
|
||||||
import com.mongodb.DBCollection
|
import com.mongodb.DBCollection
|
||||||
|
import com.fangjia.core.RegionData
|
||||||
|
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||||
|
|
||||||
class LoadGuide {
|
class LoadGuide {
|
||||||
public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017)
|
public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017)
|
||||||
@@ -14,13 +16,31 @@ class LoadGuide {
|
|||||||
}
|
}
|
||||||
|
|
||||||
def run(){
|
def run(){
|
||||||
|
// match("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
|
||||||
|
|
||||||
|
// saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞指导价218.xlsx")
|
||||||
// saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞二手房政府指导价.xls")
|
// saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞二手房政府指导价.xls")
|
||||||
// saveChengduoPrice("D:\\数据\\建行\\房产估值\\指导价\\成都部分住宅小区二手房成交参考价格201-286-258.xlsx")
|
// saveChengduoPrice("D:\\数据\\建行\\房产估值\\指导价\\成都部分住宅小区二手房成交参考价格201-286-258.xlsx")
|
||||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\绍兴指导价小区_0909.xlsx")
|
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\绍兴指导价小区_0909.xlsx")
|
||||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\宁波交易参考价格.xlsx")
|
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\宁波交易参考价格.xlsx")
|
||||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安成交参考价格.xlsx")
|
savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
|
||||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\无锡100 广州96 二手住房成交参考价格表.xlsx")
|
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\无锡100 广州96 二手住房成交参考价格表.xlsx")
|
||||||
savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx")
|
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx")
|
||||||
|
|
||||||
|
}
|
||||||
|
def match(def filename){
|
||||||
|
def city = "西安"
|
||||||
|
ExcelUtils.modify(filename,{d->
|
||||||
|
def newRegion = RegionData.formatRegion(city, d['A'])
|
||||||
|
if(newRegion){
|
||||||
|
def rMap = SeaweedDataMatchProcess.doMatch(city, newRegion, d['B'], null);
|
||||||
|
if(rMap){
|
||||||
|
d['D'] = city
|
||||||
|
d['E'] = rMap.region
|
||||||
|
d['F'] = rMap.name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},0)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -32,8 +52,8 @@ class LoadGuide {
|
|||||||
def o = new BasicDBObject()
|
def o = new BasicDBObject()
|
||||||
o.city = d['E']
|
o.city = d['E']
|
||||||
o.region = d['A']
|
o.region = d['A']
|
||||||
o.name = d['C']
|
o.name = d['B']
|
||||||
o.avg_price = d['D'].toString().toDouble().toInteger()
|
o.avg_price = d['C'].toString().toDouble().toInteger()
|
||||||
o.match_city = d['E']
|
o.match_city = d['E']
|
||||||
o.match_region = d['F']
|
o.match_region = d['F']
|
||||||
o.match_name = d['G']
|
o.match_name = d['G']
|
||||||
@@ -57,11 +77,11 @@ class LoadGuide {
|
|||||||
o.city = d['E']
|
o.city = d['E']
|
||||||
o.region = d['A']
|
o.region = d['A']
|
||||||
o.name = d['B']
|
o.name = d['B']
|
||||||
o.avg_price = d['D'].toString().toDouble().toInteger()
|
o.avg_price = d['C'].toString().toDouble().toInteger()
|
||||||
o.match_city = d['E']
|
o.match_city = d['E']
|
||||||
o.match_region = d['F']
|
o.match_region = d['F']
|
||||||
o.match_name = d['G']
|
o.match_name = d['G']
|
||||||
o.house_count = d['C'].toString().toDouble().toInteger()
|
// o.house_count = d['C'].toString().toDouble().toInteger()
|
||||||
o.source_type = 1 //1:成交参考价 2:参考价
|
o.source_type = 1 //1:成交参考价 2:参考价
|
||||||
o.c_date = new Date()
|
o.c_date = new Date()
|
||||||
rets << o
|
rets << o
|
||||||
|
|||||||
+241
-39
@@ -4,7 +4,107 @@ import com.fangjia.core.RegionData
|
|||||||
import com.fangjia.script.commData.CommonDatautils
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||||
import com.fangjia.script.util.CommonUtils
|
import com.fangjia.script.util.CommonUtils
|
||||||
|
import com.fangjia.script.util.DateUtils
|
||||||
|
import com.fangjia.script.util.DbUtils
|
||||||
import com.fangjia.script.util.ExcelUtils
|
import com.fangjia.script.util.ExcelUtils
|
||||||
|
import com.fangjia.script.util.GeoHelper
|
||||||
|
import com.fangjia.script.util.MongoUtils
|
||||||
|
import com.mongodb.BasicDBObject
|
||||||
|
import com.mongodb.DBCollection
|
||||||
|
|
||||||
|
|
||||||
|
DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
|
||||||
|
|
||||||
|
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d ->
|
||||||
|
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||||
|
// if(newRegion && !d['K']) {
|
||||||
|
// def rMap = SeaweedDataMatchProcess.doMatch(d['A'], newRegion, d['C'], d['D']+d['E']);
|
||||||
|
// if(!rMap && d['G']){
|
||||||
|
// rMap = SeaweedDataMatchProcess.doMatch(d['A'], newRegion, d['G'], d['H']);
|
||||||
|
// }
|
||||||
|
// if(rMap){
|
||||||
|
// d['O'] = rMap.city
|
||||||
|
// d['P'] = rMap.region
|
||||||
|
// d['Q'] = rMap.name
|
||||||
|
// d['R'] = rMap.address
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
//});
|
||||||
|
|
||||||
|
|
||||||
|
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\高德反查数据3_2003.xlsx",{d->
|
||||||
|
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||||
|
// if(newRegion && !d['K']){
|
||||||
|
//
|
||||||
|
// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'],"district")
|
||||||
|
// if(dist){
|
||||||
|
// d['K']="完全一致"
|
||||||
|
// d['L'] = dist.estate_type2
|
||||||
|
// d['M'] = dist.name_cat
|
||||||
|
// }else{
|
||||||
|
// dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['C'],"district")
|
||||||
|
// if(dist){
|
||||||
|
// d['K']="映射别名"
|
||||||
|
// d['L'] = dist.estate_type2
|
||||||
|
// d['M'] = dist.name_cat
|
||||||
|
// }else{
|
||||||
|
// def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['C'],"district")
|
||||||
|
// if(rets){
|
||||||
|
// dist = rets[0]
|
||||||
|
// }
|
||||||
|
// if(rets){
|
||||||
|
// d['K']="别名"
|
||||||
|
// d['L'] = dist.estate_type2
|
||||||
|
// d['M'] = dist.name_cat
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
//},0)
|
||||||
|
|
||||||
|
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\高德反查数据3_2003.xlsx",{d->
|
||||||
|
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||||
|
// if(newRegion && !d['K'] && d['G']){
|
||||||
|
//
|
||||||
|
// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['G'],"district")
|
||||||
|
// if(dist){
|
||||||
|
// d['K']="完全一致(高德)"
|
||||||
|
// d['L'] = dist.estate_type2
|
||||||
|
// d['M'] = dist.name_cat
|
||||||
|
// }else{
|
||||||
|
// dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['G'],"district")
|
||||||
|
// if(dist){
|
||||||
|
// d['K']="映射别名(高德)"
|
||||||
|
// d['L'] = dist.estate_type2
|
||||||
|
// d['M'] = dist.name_cat
|
||||||
|
// }else{
|
||||||
|
// def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['G'],"district")
|
||||||
|
// if(rets){
|
||||||
|
// dist = rets[0]
|
||||||
|
// }
|
||||||
|
// if(rets){
|
||||||
|
// d['K']="别名(高德)"
|
||||||
|
// d['L'] = dist.estate_type2
|
||||||
|
// d['M'] = dist.name_cat
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
//},0)
|
||||||
|
|
||||||
|
|
||||||
|
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\价格过滤小区融合情况_1209.xlsx",{d->
|
||||||
|
// def dist = CommonDatautils.findDistrictFieldById(d['F'], ["address","lat","lng"])
|
||||||
|
// if(dist){
|
||||||
|
// d['Z'] = dist.address
|
||||||
|
// if(dist.lat){
|
||||||
|
// d['AA'] = dist.lat/10000000000.0
|
||||||
|
// d['AB'] = dist.lng/10000000000.0
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
//},0)
|
||||||
|
|
||||||
//def cityDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\小区\\重庆高德反查数据_1529.xlsx",0,1)
|
//def cityDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\小区\\重庆高德反查数据_1529.xlsx",0,1)
|
||||||
//def cityDataMap = [:]
|
//def cityDataMap = [:]
|
||||||
@@ -74,54 +174,156 @@ def city = "
|
|||||||
//},0)
|
//},0)
|
||||||
|
|
||||||
|
|
||||||
//ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例.xlsx",{d->
|
//ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例(合并).xlsx",{d->
|
||||||
// def obj = CommonDatautils.findDistrict("广州", d['C'], d['D'], "district")
|
// def obj = CommonDatautils.findDistrict("广州", d['C'], d['D'], "district")
|
||||||
// if(obj){
|
// if(obj){
|
||||||
// def year = CommonUtils.getDistrictAge(obj)
|
//// def year = CommonUtils.getDistrictAge(obj)
|
||||||
// if(year){
|
//// if(year){
|
||||||
// d['H'] = year
|
//// d['H'] = year
|
||||||
// }
|
//// }
|
||||||
// if(obj.alias){
|
//// if(obj.alias){
|
||||||
// d['K'] = obj.alias.join(" ")
|
//// d['K'] = obj.alias.join(" ")
|
||||||
|
//// }
|
||||||
|
// if(obj.mapping_alias){
|
||||||
|
// d['X'] = obj.mapping_alias.join(" ")
|
||||||
// }
|
// }
|
||||||
//
|
//
|
||||||
// }
|
// }
|
||||||
//},0)
|
//},0)
|
||||||
|
|
||||||
def data1s = ExcelUtils.read("D:\\数据\\广发\\图片转换汇总1.8.xlsx",0, 1)
|
//def data1s = ExcelUtils.read("D:\\数据\\广发\\图片转换汇总1.8.xlsx",0, 1)
|
||||||
def scoreMap = [:]
|
//def scoreMap = [:]
|
||||||
for(def d : data1s){
|
//for(def d : data1s){
|
||||||
scoreMap.put(d['B'], d)
|
// scoreMap.put(d['B'], d)
|
||||||
}
|
//}
|
||||||
|
//
|
||||||
|
//def data2s = ExcelUtils.read("D:\\数据\\广发\\图片转换汇总1.8.xlsx",1, 1)
|
||||||
|
//def levelMap = [:]
|
||||||
|
//for(def d : data2s){
|
||||||
|
// levelMap.put(d['E'], d)
|
||||||
|
//}
|
||||||
|
//
|
||||||
|
//ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例.xlsx",{d->
|
||||||
|
// if(d['E']){
|
||||||
|
// def row = scoreMap.get(d['E'])
|
||||||
|
// if(row){
|
||||||
|
// d['I'] = row['G']
|
||||||
|
// d['J'] = row['H']
|
||||||
|
// d['K'] = row['I']
|
||||||
|
// d['L'] = row['J']
|
||||||
|
// d['M'] = row['K']
|
||||||
|
// d['N'] = row['L']
|
||||||
|
// d['O'] = row['M']
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// def data = levelMap.get(d['E'])
|
||||||
|
// if(data){
|
||||||
|
// d['P'] = data['G']
|
||||||
|
// d['Q'] = data['H']
|
||||||
|
// d['R'] = data['I']
|
||||||
|
// d['S'] = data['J']
|
||||||
|
// d['T'] = data['K']
|
||||||
|
// d['U'] = data['L']
|
||||||
|
// d['V'] = data['M']
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
//},0)
|
||||||
|
|
||||||
def data2s = ExcelUtils.read("D:\\数据\\广发\\图片转换汇总1.8.xlsx",1, 1)
|
//ExcelUtils.modify("D:\\数据\\成交\\福州成交.xlsx",{d->
|
||||||
def levelMap = [:]
|
// def o = new BasicDBObject()
|
||||||
for(def d : data2s){
|
// o.area = d['D'].toDouble()
|
||||||
levelMap.put(d['E'], d)
|
// o.avg_price = d['F'].toInteger()
|
||||||
}
|
// o.city = "福州"
|
||||||
|
// o.district_name = d['C']
|
||||||
|
// o.region = d['B']
|
||||||
|
// o.source = "贝壳找房"
|
||||||
|
// o.trade_date = DateUtils.parseFromTightYYYYMMDD(d['M'])
|
||||||
|
// o.total_price = d['E'].toInteger()*10000
|
||||||
|
// if(d['I']){
|
||||||
|
// o.hall = d['I'].toInteger()
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// o.m_date = new Date()
|
||||||
|
// if(d['H']){
|
||||||
|
// o.room = d['H'].toInteger()
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// o.create_date = new Date()
|
||||||
|
// o.status = 2
|
||||||
|
// DEAL_PRICE.save(o)
|
||||||
|
//
|
||||||
|
//},0)
|
||||||
|
|
||||||
ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例.xlsx",{d->
|
|
||||||
if(d['E']){
|
|
||||||
def row = scoreMap.get(d['E'])
|
|
||||||
if(row){
|
|
||||||
d['I'] = row['G']
|
|
||||||
d['J'] = row['H']
|
|
||||||
d['K'] = row['I']
|
|
||||||
d['L'] = row['J']
|
|
||||||
d['M'] = row['K']
|
|
||||||
d['N'] = row['L']
|
|
||||||
d['O'] = row['M']
|
|
||||||
}
|
|
||||||
|
|
||||||
def data = levelMap.get(d['E'])
|
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d ->
|
||||||
if(data){
|
// if(d['K'] && d['K']=="别名"){
|
||||||
d['P'] = data['G']
|
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||||
d['Q'] = data['H']
|
// if(newRegion){
|
||||||
d['R'] = data['I']
|
// def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['C'],"district")
|
||||||
d['S'] = data['J']
|
// def dist = null
|
||||||
d['T'] = data['K']
|
// if(rets){
|
||||||
d['U'] = data['L']
|
// dist = rets[0]
|
||||||
d['V'] = data['M']
|
// }
|
||||||
|
// if(dist){
|
||||||
|
// def alias = dist.mapping_alias
|
||||||
|
// if(!alias){
|
||||||
|
// alias = []
|
||||||
|
// }
|
||||||
|
// if(!alias.contains(d['C'])){
|
||||||
|
// alias << d['C']
|
||||||
|
// }
|
||||||
|
// def udata = new BasicDBObject()
|
||||||
|
// udata.put("mapping_alias",alias)
|
||||||
|
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
//});
|
||||||
|
|
||||||
|
|
||||||
|
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d ->
|
||||||
|
// if(d['K'] && d['K']=="映射别名(高德)" && d['G']){
|
||||||
|
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||||
|
// if(newRegion){
|
||||||
|
// dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['G'],"district")
|
||||||
|
// if(dist){
|
||||||
|
// def alias = dist.mapping_alias
|
||||||
|
// if(!alias){
|
||||||
|
// alias = []
|
||||||
|
// }
|
||||||
|
// if(!alias.contains(d['C'])){
|
||||||
|
// alias << d['C']
|
||||||
|
// }
|
||||||
|
// def udata = new BasicDBObject()
|
||||||
|
// udata.put("mapping_alias",alias)
|
||||||
|
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
//});
|
||||||
|
|
||||||
|
|
||||||
|
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d ->
|
||||||
|
if(d['K'] && d['K']=="别名(高德)" && d['G']){
|
||||||
|
def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||||
|
if(newRegion){
|
||||||
|
def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['G'],"district")
|
||||||
|
def dist = null
|
||||||
|
if(rets){
|
||||||
|
dist = rets[0]
|
||||||
|
}
|
||||||
|
if(dist){
|
||||||
|
def alias = dist.mapping_alias
|
||||||
|
if(!alias){
|
||||||
|
alias = []
|
||||||
|
}
|
||||||
|
if(!alias.contains(d['C'])){
|
||||||
|
alias << d['C']
|
||||||
|
}
|
||||||
|
def udata = new BasicDBObject()
|
||||||
|
udata.put("mapping_alias",alias)
|
||||||
|
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},0)
|
});
|
||||||
|
|||||||
+12
@@ -27,6 +27,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
|||||||
def citys = CcbCommonUtils.citys;
|
def citys = CcbCommonUtils.citys;
|
||||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||||
|
|
||||||
|
//检查ok文件是否上传
|
||||||
// exportAllBigdataDealFJW()
|
// exportAllBigdataDealFJW()
|
||||||
for(def city : citys) {
|
for(def city : citys) {
|
||||||
def cityCode = cityCodeDataMap.get(city)
|
def cityCode = cityCodeDataMap.get(city)
|
||||||
@@ -47,6 +48,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
|||||||
exportBigdataCommlayoutpicFjw(cityCode)
|
exportBigdataCommlayoutpicFjw(cityCode)
|
||||||
}
|
}
|
||||||
exportEvaluateRate()
|
exportEvaluateRate()
|
||||||
|
exportOk()
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -593,4 +595,14 @@ class ExportDistrictRongHe extends BaseBatch{
|
|||||||
println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......"
|
println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def exportOk(){
|
||||||
|
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||||
|
def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok"
|
||||||
|
if(CommonDatautils.isWindow()){
|
||||||
|
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok"
|
||||||
|
}
|
||||||
|
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
|
||||||
|
CcbFtpUtils.ftpUpload(dataCodeFileName, curDate)
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
+28
-1
@@ -13,7 +13,34 @@ class DownPdfCheck {
|
|||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new DownPdfCheck();
|
def task = new DownPdfCheck();
|
||||||
task.run();
|
// task.run();
|
||||||
|
|
||||||
|
task.czLogRun()
|
||||||
|
}
|
||||||
|
|
||||||
|
def czLogRun(){
|
||||||
|
def files = getAllFiles("D:\\数据\\常州")
|
||||||
|
if(files){
|
||||||
|
for(def filepath : files){
|
||||||
|
read(filepath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def read(file){
|
||||||
|
// File file = new File(fileName);
|
||||||
|
def index = 0
|
||||||
|
|
||||||
|
file.eachLine{it->
|
||||||
|
if(it.contains("4ead0347b4055be6989e726e")||it.contains("61a2e0")){
|
||||||
|
index++
|
||||||
|
|
||||||
|
println it
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
def run(){
|
def run(){
|
||||||
|
|||||||
+28
@@ -8,6 +8,7 @@ import com.fangjia.script.common.DataHandleUtils
|
|||||||
import com.fangjia.script.jusfoun.JusfounCommon
|
import com.fangjia.script.jusfoun.JusfounCommon
|
||||||
import com.fangjia.script.util.DateUtils
|
import com.fangjia.script.util.DateUtils
|
||||||
import org.apache.commons.codec.digest.DigestUtils
|
import org.apache.commons.codec.digest.DigestUtils
|
||||||
|
import org.apache.commons.io.FileUtils
|
||||||
|
|
||||||
import java.util.concurrent.TimeUnit
|
import java.util.concurrent.TimeUnit
|
||||||
|
|
||||||
@@ -79,6 +80,14 @@ class DistrictPriceExportBatch extends BaseBatch{
|
|||||||
def lastDate = new Date()
|
def lastDate = new Date()
|
||||||
def startDate = DateUtils.getFirstDayOfMonth(lastDate, -1)
|
def startDate = DateUtils.getFirstDayOfMonth(lastDate, -1)
|
||||||
|
|
||||||
|
// List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202111.csv"))
|
||||||
|
// def map2 = [:]
|
||||||
|
// for(def s : list2) {
|
||||||
|
// def row = s.split("@\\|@")
|
||||||
|
// def key = row[2] + "##" + row[3] + "##" + row[4]
|
||||||
|
// map2.put(key, row[5])
|
||||||
|
// }
|
||||||
|
|
||||||
def result = []
|
def result = []
|
||||||
for(def rMap : districts){
|
for(def rMap : districts){
|
||||||
def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate)
|
def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate)
|
||||||
@@ -90,6 +99,25 @@ class DistrictPriceExportBatch extends BaseBatch{
|
|||||||
m << rMap.region
|
m << rMap.region
|
||||||
m << rMap.name
|
m << rMap.name
|
||||||
m << dbo.get("avg_price")
|
m << dbo.get("avg_price")
|
||||||
|
// def lastMonthPrice = map2.get(rMap.city + "##" + rMap.region+"##"+rMap.name)
|
||||||
|
// if(lastMonthPrice && dbo.get("avg_price")){
|
||||||
|
// lastMonthPrice = lastMonthPrice.toInteger()
|
||||||
|
// def markup = (dbo.get("avg_price")-lastMonthPrice)*100.0/lastMonthPrice
|
||||||
|
// if(Math.abs(markup)>10){
|
||||||
|
// def random = 0
|
||||||
|
// if(dbo.get("avg_price")-lastMonthPrice>=0){
|
||||||
|
// random = com.fangjia.core.util.RandomUtil.random(1, 9)
|
||||||
|
// }else{
|
||||||
|
// random = com.fangjia.core.util.RandomUtil.random(-9, 0)
|
||||||
|
// }
|
||||||
|
// def newPrice = lastMonthPrice*random/100 + lastMonthPrice
|
||||||
|
// m << newPrice.toInteger()
|
||||||
|
// }else{
|
||||||
|
// m << dbo.get("avg_price")
|
||||||
|
// }
|
||||||
|
// }else{
|
||||||
|
// m << dbo.get("avg_price")
|
||||||
|
// }
|
||||||
m << curDate.toInteger();
|
m << curDate.toInteger();
|
||||||
result << m
|
result << m
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -48,51 +48,51 @@ import org.bson.types.ObjectId
|
|||||||
// return location;
|
// return location;
|
||||||
//}
|
//}
|
||||||
|
|
||||||
//List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202109.csv"))
|
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202112.csv"))
|
||||||
//def map1 = [:]
|
def map1 = [:]
|
||||||
//for(def s : list1){
|
for(def s : list1){
|
||||||
// def row = s.split("@\\|@")
|
def row = s.split("@\\|@")
|
||||||
// def key = row[2]+"##"+row[3]+"##"+row[4]
|
def key = row[2]+"##"+row[3]+"##"+row[4]
|
||||||
// map1.put(key, row[5])
|
map1.put(key, row[5])
|
||||||
//}
|
}
|
||||||
//
|
|
||||||
//List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202108.csv"))
|
List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202111.csv"))
|
||||||
//def map2 = [:]
|
def map2 = [:]
|
||||||
//for(def s : list2) {
|
for(def s : list2) {
|
||||||
// def row = s.split("@\\|@")
|
def row = s.split("@\\|@")
|
||||||
// def key = row[2] + "##" + row[3] + "##" + row[4]
|
def key = row[2] + "##" + row[3] + "##" + row[4]
|
||||||
// map2.put(key, row[5])
|
map2.put(key, row[5])
|
||||||
//}
|
}
|
||||||
//def rows = []
|
def rows = []
|
||||||
//map1.each {k,v->
|
map1.each {k,v->
|
||||||
// def row = []
|
def row = []
|
||||||
// def keys = k.split("##")
|
def keys = k.split("##")
|
||||||
// def lastPrice = map2.get(k)
|
def lastPrice = map2.get(k)
|
||||||
// rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString()]
|
rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString()]
|
||||||
//}
|
}
|
||||||
//ExcelUtils.write("D:\\数据\\福州\\价格比对.xlsx", rows, "数据")
|
ExcelUtils.write("D:\\数据\\福州\\价格比对12-2.xlsx", rows, "数据")
|
||||||
ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州涨幅在10%以上小区.xlsx",{d->
|
//ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州涨幅在10%以上小区.xlsx",{d->
|
||||||
def obj = CommonDatautils.queryDistrictPrice(d['A'],d['B'], d['C'], DateUtils.parseFromTightByPattern("2021-10-10","yyyy-MM-dd"))
|
// def obj = CommonDatautils.queryDistrictPrice(d['A'],d['B'], d['C'], DateUtils.parseFromTightByPattern("2021-10-10","yyyy-MM-dd"))
|
||||||
if(obj){
|
// if(obj){
|
||||||
if(obj.value_type == 1){
|
// if(obj.value_type == 1){
|
||||||
d["G"] = "指导价"
|
// d["G"] = "指导价"
|
||||||
}else if(obj.value_type == 2){
|
// }else if(obj.value_type == 2){
|
||||||
d["G"] ="自身样本"
|
// d["G"] ="自身样本"
|
||||||
}else if(obj.value_type == 3){
|
// }else if(obj.value_type == 3){
|
||||||
d["G"] = "第三方"
|
// d["G"] = "第三方"
|
||||||
}else if(obj.value_type == 4){
|
// }else if(obj.value_type == 4){
|
||||||
d["G"] = "交易数据"
|
// d["G"] = "交易数据"
|
||||||
}else if(obj.value_type == 5){
|
// }else if(obj.value_type == 5){
|
||||||
d["G"] ="周边楼盘"
|
// d["G"] ="周边楼盘"
|
||||||
}else if(obj.value_type == 6){
|
// }else if(obj.value_type == 6){
|
||||||
d["G"] ="推导"
|
// d["G"] ="推导"
|
||||||
}else if(obj.value_type == 9){
|
// }else if(obj.value_type == 9){
|
||||||
d["G"] = "神经网络"
|
// d["G"] = "神经网络"
|
||||||
}else if(obj.value_type == 10){
|
// }else if(obj.value_type == 10){
|
||||||
d["G"] = "无方式"
|
// d["G"] = "无方式"
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
},0)
|
//},0)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
|||||||
//福州第三方均价统计,发送钉钉消息
|
//福州第三方均价统计,发送钉钉消息
|
||||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202111.csv")
|
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202112.csv")
|
||||||
def count0 = 0
|
def count0 = 0
|
||||||
def count1 = 0
|
def count1 = 0
|
||||||
def count2 = 0
|
def count2 = 0
|
||||||
|
|||||||
+26
-16
@@ -62,7 +62,8 @@ class DistrictIndexBatch extends BaseBatch{
|
|||||||
|
|
||||||
def run(){
|
def run(){
|
||||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||||
def citys = ["上海"]//CommonDatautils.findAllCity();
|
def citys = CommonDatautils.findAllCity();
|
||||||
|
citys.remove("上海")
|
||||||
for(def city : citys){
|
for(def city : citys){
|
||||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2"])
|
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2"])
|
||||||
// dists = dists[0..20]
|
// dists = dists[0..20]
|
||||||
@@ -156,13 +157,19 @@ class DistrictIndexBatch extends BaseBatch{
|
|||||||
fileName = "D:\\数据\\评级\\上海综合潜力.dat"
|
fileName = "D:\\数据\\评级\\上海综合潜力.dat"
|
||||||
}
|
}
|
||||||
|
|
||||||
FileUtils.batchCreateFile(datas, fileName, headers, "utf-8", false);
|
FileUtils.batchCreateFile(datas, fileName, headers, "utf-8", true);
|
||||||
// ExcelUtils.write("D:\\数据\\评级\\上海综合潜力.xlsx", datas, "数据")
|
// ExcelUtils.write("D:\\数据\\评级\\上海综合潜力.xlsx", datas, "数据")
|
||||||
}
|
}
|
||||||
|
|
||||||
def getHousePrice(d, dists, priceMap, citycode){
|
def getHousePrice(d, dists, priceMap, citycode){
|
||||||
//def areas = findHouse(d, 20200101, 20220101)
|
//def areas = findHouse(d, 20200101, 20220101)
|
||||||
def areas = findHouseByOracle(d._id.toString(), citycode);
|
|
||||||
|
def areas = []
|
||||||
|
if(citycode){
|
||||||
|
areas = findHouseByOracle(d._id.toString(), citycode);
|
||||||
|
}else{
|
||||||
|
areas = findHouse(d, 20200101, 20220101)
|
||||||
|
}
|
||||||
def avg_area = null
|
def avg_area = null
|
||||||
if(areas){
|
if(areas){
|
||||||
avg_area = areas.sum()/areas.size();
|
avg_area = areas.sum()/areas.size();
|
||||||
@@ -188,24 +195,27 @@ class DistrictIndexBatch extends BaseBatch{
|
|||||||
|
|
||||||
if(selectDists){
|
if(selectDists){
|
||||||
println d.region+" "+d.name+" 周边同建筑年代楼盘:"+selectDists.size()
|
println d.region+" "+d.name+" 周边同建筑年代楼盘:"+selectDists.size()
|
||||||
// def allAreas = []
|
def allAreas = []
|
||||||
def ids = []
|
def ids = []
|
||||||
for(def sd : selectDists){
|
if(citycode){
|
||||||
// //def areaDatas = findHouse(sd, 20190101, 20220101
|
for(def sd : selectDists){
|
||||||
// def areaDatas = findHouseByOracle(sd._id.toString());
|
ids << sd._id.toString()
|
||||||
// allAreas.addAll(areaDatas)
|
}
|
||||||
ids << sd._id.toString()
|
if(ids.size()>=901){
|
||||||
}
|
ids = ids[0..900]
|
||||||
if(ids.size()>=901){
|
}
|
||||||
ids = ids[0..900]
|
|
||||||
}
|
|
||||||
|
|
||||||
def idstr = ids.join("','")
|
def idstr = ids.join("','")
|
||||||
def allAreas = findHouseByOracle2(idstr,citycode);
|
allAreas = findHouseByOracle2(idstr,citycode);
|
||||||
|
}else{
|
||||||
|
for(def sd : selectDists){
|
||||||
|
def areaDatas = findHouse(sd, 20190101, 20220101)
|
||||||
|
allAreas.addAll(areaDatas)
|
||||||
|
}
|
||||||
|
}
|
||||||
if(allAreas){
|
if(allAreas){
|
||||||
avg_area = allAreas.sum()/allAreas.size();
|
avg_area = allAreas.sum()/allAreas.size();
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+7
-3
@@ -98,9 +98,9 @@ class CrawalUtil {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
static def getDocumentByABuYun(def url, def errorMsg, def charset, def language = null) {
|
static def getDocumentByABuYun(def url, def errorMsg, def charset, def language = null, def isJson=false) {
|
||||||
def flag = true;
|
def flag = true;
|
||||||
Document document = null;
|
def document = null;
|
||||||
def count = 0;
|
def count = 0;
|
||||||
while (flag) {
|
while (flag) {
|
||||||
def content = null;
|
def content = null;
|
||||||
@@ -121,7 +121,11 @@ class CrawalUtil {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (content) {
|
if (content) {
|
||||||
document = Jsoup.parse(content);
|
if(!isJson){
|
||||||
|
document = Jsoup.parse(content);
|
||||||
|
}else{
|
||||||
|
document = content
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if (!content || (content && errorMsg && content.contains(errorMsg))) {
|
if (!content || (content && errorMsg && content.contains(errorMsg))) {
|
||||||
count++;
|
count++;
|
||||||
|
|||||||
+1
-1
@@ -364,7 +364,7 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
def standardDistrict(def data, def avg_price){
|
def standardDistrict(def data, def avg_price){
|
||||||
//名称匹配,名称别名匹配,地址别名匹配
|
//名称匹配,名称别名匹配,地址别名匹配
|
||||||
//def district = getMatchName(data.city, data.region, data.name, data.address,"district")
|
//def district = getMatchName(data.city, data.region, data.name, data.address,"district")
|
||||||
data.name = data.name.replace("(","").replace(")","")
|
data.name = data.name.replace("(","").replace(")","").replace("数据由万科物业提供","").trim()
|
||||||
def district = null;
|
def district = null;
|
||||||
def type = 0;
|
def type = 0;
|
||||||
if(data.name.contains("写字楼")||data.name.contains("办公楼")||data.name.contains("商业")){
|
if(data.name.contains("写字楼")||data.name.contains("办公楼")||data.name.contains("商业")){
|
||||||
|
|||||||
+4
-4
@@ -40,11 +40,11 @@ class DataStat {
|
|||||||
|
|
||||||
|
|
||||||
def run(){
|
def run(){
|
||||||
def source = "城市房产"//
|
def source = "58同城"//
|
||||||
def citys = findCrawlCity(source, 202110);
|
def citys = findCrawlCity(source, 202111);
|
||||||
for(def city : citys){
|
for(def city : citys){
|
||||||
def count1 = findCount(source, city, 202110);
|
def count1 = findCount(source, city, 202111);
|
||||||
def count2 = findCount(source, city, 202111);
|
def count2 = findCount(source, city, 202112);
|
||||||
println city +" "+count1+" "+count2
|
println city +" "+count1+" "+count2
|
||||||
// println count1
|
// println count1
|
||||||
// println "--- ----"
|
// println "--- ----"
|
||||||
|
|||||||
+55
-18
File diff suppressed because one or more lines are too long
+2
-2
@@ -17,8 +17,8 @@ class SynchDistrictPrice {
|
|||||||
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
|
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
|
||||||
// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
|
// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
|
||||||
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
|
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
|
||||||
def sourceMap = ["58.com":"58同城","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多",
|
//"fang.com":"搜房",
|
||||||
"anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
|
def sourceMap = ["fang.com":"搜房"]
|
||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new SynchDistrictPrice()
|
def task = new SynchDistrictPrice()
|
||||||
task.run()
|
task.run()
|
||||||
|
|||||||
Reference in New Issue
Block a user