+21
-3
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
|
||||
/**
|
||||
* 深圳公众号成交数据录入
|
||||
*/
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳4月份二手房成交数据汇总.xls", 0, 1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳5月份二手房成交数据汇总.xls", 0, 1)
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
@@ -51,7 +51,19 @@ for(def d : datas){
|
||||
map.region = d['B']
|
||||
def roomStr = d['E']
|
||||
if(roomStr){
|
||||
map.room = roomStr.toString().toDouble().toInteger()
|
||||
if(roomStr.contains("-")){
|
||||
def arrs = roomStr.split("-")
|
||||
try{
|
||||
map.room = arrs[0].toString().toInteger()
|
||||
}catch(Exception e){
|
||||
println "room error"
|
||||
}
|
||||
map.hall = arrs[1].toString().toInteger()
|
||||
map.toilet = arrs[3].toString().toInteger()
|
||||
}else{
|
||||
map.room = roomStr.toString().toDouble().toInteger()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
map.area = d['F'].toString().toDouble()
|
||||
@@ -59,7 +71,13 @@ for(def d : datas){
|
||||
map.avg_price = (int)(d['H'].toString().toDouble())
|
||||
map.sell_price = (int)(d['I'].toString().toDouble()*10000)
|
||||
def trade_date = d['J']
|
||||
map.trade_date = trade_date
|
||||
if(trade_date){
|
||||
map.trade_date = trade_date
|
||||
}else{
|
||||
def trade_date1 = DateUtils.getRandomDate("2023-05-16", "2023-05-31")
|
||||
map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
|
||||
}
|
||||
|
||||
map.source = "深房家族"
|
||||
map.create_date = new Date()
|
||||
map.m_date = new Date()
|
||||
|
||||
+4
-5
@@ -56,8 +56,8 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
|
||||
|
||||
def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用
|
||||
// def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
|
||||
// def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用
|
||||
def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
|
||||
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
|
||||
rets = addDeleteDist(dists, ccbMaps, rets)
|
||||
rows.addAll(rets)
|
||||
@@ -77,9 +77,8 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
println "txt 生成->导Oracle"
|
||||
|
||||
|
||||
// delPgData()
|
||||
// saveToPgSql(rows)
|
||||
delPgData()
|
||||
saveToPgSql(rows)
|
||||
|
||||
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
|
||||
// def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat")
|
||||
|
||||
+3
-3
@@ -61,9 +61,9 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1)
|
||||
|
||||
def lastYearDate = DateUtils.addMonth(startDate.toString(),-12, "yyyyMMdd").toInteger();
|
||||
cityRun(citys, startDate, endDate, cityCodeMap)
|
||||
regionRun(citys, startDate, endDate, cityCodeMap)
|
||||
// districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap)
|
||||
// cityRun(citys, startDate, endDate, cityCodeMap)
|
||||
// regionRun(citys, startDate, endDate, cityCodeMap)
|
||||
districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap)
|
||||
}
|
||||
|
||||
def cityRun(citys, startDate, endDate, cityCodeDataMap){
|
||||
|
||||
+64
-2
@@ -40,19 +40,25 @@ class HailanTest {
|
||||
static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new HailanTest();
|
||||
task.run();
|
||||
}
|
||||
|
||||
def run(){
|
||||
def inputfile = "D:\\数据\\微元素\\排查表.xlsx"
|
||||
match(inputfile)
|
||||
def inputfile = "D:\\数据\\微元素\\test.xlsx"
|
||||
// addThirdPrice(inputfile)
|
||||
// match(inputfile)
|
||||
// tradeAvgPrice(inputfile)
|
||||
// caluateTotalPrice(inputfile)
|
||||
// addPrice(inputfile)
|
||||
// addCommId("D:\\数据\\海南\\匹配上的合并数据.xlsx")
|
||||
// addHouse(inputfile)
|
||||
// stat(inputfile)
|
||||
// hailanStat(inputfile)
|
||||
addFzFjwPrice(inputfile)
|
||||
}
|
||||
|
||||
def addHouse(inputfile){
|
||||
@@ -702,4 +708,60 @@ class HailanTest {
|
||||
}
|
||||
return prices;
|
||||
}
|
||||
|
||||
def tradeAvgPrice(inputfile){
|
||||
ExcelUtils.modify(inputfile, { row ->
|
||||
if(row['I'] && row['I'] == "4f4dac0a84ae1852a920ad39"){
|
||||
def dist = CommonDatautils.findDistrictById(row['I'])
|
||||
if(dist){
|
||||
def tradeAvgPrice = CommonDatautils.findTrades(dist, "2023-02-01", "2023-06-01");
|
||||
row['J'] = (int)tradeAvgPrice
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def addThirdPrice(path){
|
||||
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302
|
||||
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
|
||||
|
||||
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
def cityMap = CommonDatautils.findThirdPartyPrice("福州", row['F'], row['G'], startDate, endDate)
|
||||
def priceMap = cityMap.get("福州"+"##"+row['F']+"##"+row['G'])
|
||||
if(priceMap){
|
||||
if(priceMap.containsKey("lianjia_esf_price")){
|
||||
row['K'] = (int)(priceMap.get("lianjia_esf_price")*0.95)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def caluateTotalPrice(inputfile){
|
||||
ExcelUtils.multiThreadingModify(inputfile, { row ->
|
||||
if(row['J']){
|
||||
row['L'] = (int)(row['J'].toString().toDouble()*row['B'].toString().toDouble())
|
||||
}else if(row['K']){
|
||||
row['L'] = (int)(row['K'].toString().toDouble()*row['B'].toString().toDouble())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addFzFjwPrice(inputfile){
|
||||
ExcelUtils.modify(inputfile, { row ->
|
||||
if(row['B']) {
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", "福州")
|
||||
q.put("region", row['A'])
|
||||
q.put("name", row['B'])
|
||||
q.put("month", 202304)
|
||||
def obj = FZ_DISTRICT_STAT_COLL.findOne(q)
|
||||
if (obj) {
|
||||
row['D'] = obj.avg_price
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+70
-1
@@ -1022,7 +1022,7 @@ class CommonDatautils {
|
||||
q.put("cat", cat);
|
||||
q.put("region", region)
|
||||
q.put("name", name);
|
||||
q.put("status", 0)
|
||||
q.put("visible", 0)
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -3801,5 +3801,74 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
static def findTrades(dist, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
q.put("district_name", new BasicDBObject('$in', dist.alias))
|
||||
}else{
|
||||
q.put("district_name", dist.name)
|
||||
}
|
||||
|
||||
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
def avg_price = 0
|
||||
def total_price = 0
|
||||
def count = 0
|
||||
|
||||
for(def d : dbcorsor){
|
||||
if(d.avg_price){
|
||||
total_price = total_price + d.avg_price
|
||||
count++
|
||||
}
|
||||
}
|
||||
if(count > 3){
|
||||
avg_price = total_price/count
|
||||
}
|
||||
return avg_price
|
||||
}
|
||||
|
||||
static def findThirdPartyPrice(def city, def region, def name, def startDate, def endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||
def dbcursor = thirdpartColl.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def map = [:]
|
||||
for(def db : dbcursor){
|
||||
def key = db.city+"##"+db.region+"##"+db.name
|
||||
def dataMap = map.get(key)
|
||||
if(!dataMap){
|
||||
dataMap = [:]
|
||||
map.put(key, dataMap)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
|
||||
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
|
||||
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
|
||||
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
|
||||
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("lianjia_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
|
||||
dataMap.put("lianjia_esf_price", db.beike_esf_price)
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
+1
-1
@@ -25,7 +25,7 @@ class FzDataResultStat {
|
||||
static main(args) {
|
||||
def task = new FzDataResultStat()
|
||||
task.priceMarkupRun();
|
||||
// task.regionPriceTrendRun()
|
||||
task.regionPriceTrendRun()
|
||||
//// task.margeExcel()
|
||||
// task.saveCsvData()
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
||||
//福州第三方均价统计,发送钉钉消息
|
||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202304.csv")
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202305.csv")
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
|
||||
+2
-2
@@ -48,11 +48,11 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2023-04-01"
|
||||
static def statDate = "2023-05-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
// task.esfParser("D:\\数据\\友达\\2023年5月\\查询1.DBF");//友达二手房存入mongo
|
||||
// task.esfParser("D:\\数据\\友达\\2023年6月\\查询1.DBF");//友达二手房存入mongo
|
||||
// task.newhouseToSaveEs();//新房保存到es
|
||||
task.esfhouseToSaveEs();//二手房保存到es
|
||||
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
|
||||
|
||||
+1
-1
@@ -65,7 +65,7 @@ class AddDistrictElevatorBatch extends BaseBatch{
|
||||
}
|
||||
def newHouses = analyElevator(haveHouses, dist.estate_type2)
|
||||
if(newHouses && newHouses.size()>=3 && newHouses.size()>=haveHouses.size()*0.6){
|
||||
println totalCount.get()+" [${dist.estate_type2}] "+dist.name +" 原:"+houses.size()+" 有值:"+haveHouses.size()+" 10层以上:"+newHouses.size()
|
||||
println totalCount.get()+" [${dist.estate_type2}] "+dist.name +" 原:"+houses.size()+" 有值:"+haveHouses.size()+" 筛选样本量:"+newHouses.size()
|
||||
BasicDBObject udata = new BasicDBObject()
|
||||
if(dist.estate_type2 == "别墅"){
|
||||
udata.put("is_elevator", false)
|
||||
|
||||
+17
-11
@@ -289,20 +289,26 @@ public class BeiKeCrawlByChrome {
|
||||
}
|
||||
detail.source = "贝壳找房"
|
||||
|
||||
def liftStr = JQueryUtils.selectText("li:contains(梯户比例)",detailDoc).replace("梯户比例","")
|
||||
def detailDiv1 = JQueryUtils.select(".introContent .base", detailDoc)
|
||||
def detailDiv2 = JQueryUtils.select(".introContent .transaction", detailDoc)
|
||||
|
||||
def liftStr = detailDiv1.select("li:contains(梯户比例)").text().replace("梯户比例","")
|
||||
if(liftStr){
|
||||
detail.lift = DataFormatUtils.parseLift(liftStr)
|
||||
detail.lift_household = DataFormatUtils.parseHousehold(liftStr)
|
||||
}
|
||||
detail.trade_property = JQueryUtils.selectText("li:contains(交易权属)", detailDoc).replace("交易权属","")
|
||||
detail.room_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(户型结构)", detailDoc).replace("户型结构",""))
|
||||
detail.property_ownership = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房权所属)", detailDoc).replace("房权所属",""))
|
||||
detail.housing_age = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房屋年限)", detailDoc).replace("房屋年限",""))
|
||||
detail.elevator = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(配备电梯)", detailDoc).replace("配备电梯",""))
|
||||
detail.build_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑结构)", detailDoc).replace("建筑结构",""))
|
||||
detail.build_year = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建成年代)", detailDoc).replace("建成年代",""))
|
||||
detail.build_type = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑类型)", detailDoc).replace("建筑类型",""))
|
||||
def floorStr = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(所在楼层)", detailDoc).replace("㎡",""))
|
||||
detail.room_structure = DataFormatUtils.getValue(detailDiv1.select("li:contains(户型结构)").text().replace("户型结构",""))
|
||||
detail.elevator = DataFormatUtils.getValue(detailDiv1.select("li:contains(配备电梯)").text().replace("配备电梯",""))
|
||||
detail.build_structure = DataFormatUtils.getValue(detailDiv1.select("li:contains(建筑结构)").text().replace("建筑结构",""))
|
||||
detail.build_year = DataFormatUtils.getValue(detailDiv1.select("li:contains(建成年代)").text().replace("建成年代",""))
|
||||
detail.build_type = DataFormatUtils.getValue(detailDiv1.select("li:contains(建筑类型)").text().replace("建筑类型",""))
|
||||
def floorStr = DataFormatUtils.getValue(detailDiv1.select("li:contains(所在楼层)").text().replace("㎡",""))
|
||||
|
||||
detail.trade_property = detailDiv2.select("li:contains(交易权属)").text().replace("交易权属","")
|
||||
detail.property_ownership = DataFormatUtils.getValue(detailDiv2.select("li:contains(房权所属)").text().replace("房权所属",""))
|
||||
detail.house_use = DataFormatUtils.getValue(detailDiv2.select("li:contains(房屋用途)").text().replace("房屋用途",""))
|
||||
detail.house_use = DataFormatUtils.getValue(detailDiv2.select("li:contains(房屋用途)").text().replace("房屋用途",""))
|
||||
detail.housing_age = DataFormatUtils.getValue(detailDiv2.select("li:contains(房屋年限)").text().replace("房屋年限",""))
|
||||
if(detail.total_price && detail.avg_price){
|
||||
|
||||
detail.url = detailUrl
|
||||
@@ -483,7 +489,7 @@ public class BeiKeCrawlByChrome {
|
||||
elemPassword.sendKeys("12qwaszx");
|
||||
|
||||
btn.submit(); // 点击行为;
|
||||
Thread.sleep(20000)
|
||||
Thread.sleep(10000)
|
||||
}catch(Exception e){
|
||||
Thread.sleep(15000)
|
||||
}
|
||||
|
||||
+30
-18
@@ -297,16 +297,23 @@ public class LianjiaCrawlByChrome {
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
def dbo = findTrade(durl , "链家在线")
|
||||
if (!dbo) {
|
||||
parse(city, e, regionText, driver)
|
||||
if(dealDate){
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
println "1..."+new Date()
|
||||
def dbo = findTrade(durl , "链家在线")
|
||||
println "2..."+new Date()
|
||||
if (!dbo) {
|
||||
parse(city, e, regionText, driver)
|
||||
}
|
||||
}else{
|
||||
stop = true;
|
||||
break;
|
||||
}
|
||||
}else{
|
||||
stop = true;
|
||||
break;
|
||||
println url +" 成交时间有误!"
|
||||
}
|
||||
|
||||
}
|
||||
if(page>=100){
|
||||
stop = true;
|
||||
@@ -326,8 +333,9 @@ public class LianjiaCrawlByChrome {
|
||||
if(price){
|
||||
def detailUrl = e.select(".title a").attr("href")
|
||||
driver.get(detailUrl)
|
||||
println "3..."+new Date()
|
||||
def detailDoc = checkDocument(driver, detailUrl);
|
||||
|
||||
println "4..."+new Date()
|
||||
def title = e.select(".title a").text()
|
||||
if (title) {
|
||||
def detail = new BasicDBObject()
|
||||
@@ -386,20 +394,22 @@ public class LianjiaCrawlByChrome {
|
||||
detail.source = "链家在线"
|
||||
detail.url = detailUrl
|
||||
|
||||
def liftStr = JQueryUtils.selectText("li:contains(梯户比例)",detailDoc).replace("梯户比例","")
|
||||
def detailDiv = JQueryUtils.select(".introContent", detailDoc)
|
||||
|
||||
def liftStr = detailDiv.select("li:contains(梯户比例)").text().replace("梯户比例","")
|
||||
if(liftStr){
|
||||
detail.lift = DataFormatUtils.parseLift(liftStr)
|
||||
detail.lift_household = DataFormatUtils.parseHousehold(liftStr)
|
||||
}
|
||||
detail.trade_property = JQueryUtils.selectText("li:contains(交易权属)", detailDoc).replace("交易权属","")
|
||||
detail.room_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(户型结构)", detailDoc).replace("户型结构",""))
|
||||
detail.property_ownership = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房权所属)", detailDoc).replace("房权所属",""))
|
||||
detail.housing_age = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房屋年限)", detailDoc).replace("房屋年限",""))
|
||||
detail.elevator = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(配备电梯)", detailDoc).replace("配备电梯",""))
|
||||
detail.build_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑结构)", detailDoc).replace("建筑结构",""))
|
||||
detail.build_year = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建成年代)", detailDoc).replace("建成年代",""))
|
||||
detail.build_type = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑类型)", detailDoc).replace("建筑类型",""))
|
||||
def floorStr = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(所在楼层)", detailDoc).replace("所在楼层",""))
|
||||
detail.trade_property = detailDiv.select("li:contains(交易权属)").text().replace("交易权属","")
|
||||
detail.room_structure = DataFormatUtils.getValue( detailDiv.select("li:contains(户型结构)").text().replace("户型结构",""))
|
||||
detail.property_ownership = DataFormatUtils.getValue( detailDiv.select("li:contains(房权所属)").text().replace("房权所属",""))
|
||||
detail.housing_age = DataFormatUtils.getValue( detailDiv.select("li:contains(房屋年限)").text().replace("房屋年限",""))
|
||||
detail.elevator = DataFormatUtils.getValue( detailDiv.select("li:contains(配备电梯)").text().replace("配备电梯",""))
|
||||
detail.build_structure = DataFormatUtils.getValue( detailDiv.select("li:contains(建筑结构)").text().replace("建筑结构",""))
|
||||
detail.build_year = DataFormatUtils.getValue( detailDiv.select("li:contains(建成年代)").text().replace("建成年代",""))
|
||||
detail.build_type = DataFormatUtils.getValue( detailDiv.select("li:contains(建筑类型)").text().replace("建筑类型",""))
|
||||
def floorStr = DataFormatUtils.getValue( detailDiv.select("li:contains(所在楼层)").text().replace("所在楼层",""))
|
||||
|
||||
if(detail.total_price && detail.avg_price){
|
||||
def houseInfo = e.select(".houseInfo").text()
|
||||
@@ -417,7 +427,9 @@ public class LianjiaCrawlByChrome {
|
||||
detail.create_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
println detail
|
||||
println "5..."+new Date()
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
println "6..."+new Date()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+12
-3
@@ -28,9 +28,18 @@ class DayUpdateDistRefreshBatch extends BaseBatch{
|
||||
|
||||
if(dists){
|
||||
for(def d : dists){
|
||||
def distObj = CommonDatautils.findDistrictByParams(d.city, d.region, d.name, "district", fields)
|
||||
def distObj = null
|
||||
if(d.comm_id){
|
||||
distObj = CommonDatautils.findDistrictById(d.comm_id)
|
||||
}else{
|
||||
distObj = CommonDatautils.findDistrictByParams(d.city, d.region, d.name, "district", fields)
|
||||
}
|
||||
if(distObj && distObj._id){
|
||||
FzDataUtils.converDistrict(distObj, null, null);
|
||||
if(d.type && d.type==1){
|
||||
FzDataUtils.removeById(d.comm_id)
|
||||
}else{
|
||||
FzDataUtils.converDistrict(distObj, null, null);
|
||||
}
|
||||
println new Date().toString()+" 【处理完成】 "+d.city +" "+d.region+" "+d.name
|
||||
updateTaskStatus(d.id, 2);
|
||||
}
|
||||
@@ -57,7 +66,7 @@ class DayUpdateDistRefreshBatch extends BaseBatch{
|
||||
def q = new BasicDBObject()
|
||||
q.put("status", 5);//未处理
|
||||
TASK_COLL.find(q).each {d->
|
||||
rets << ["id":d._id.toString(),"city":d.city, "region":d.region, "name":d.name]
|
||||
rets << ["id":d._id.toString(),"city":d.city, "region":d.region, "name":d.name, "type":d.type, "comm_id":d.comm_id]
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
+7
@@ -119,6 +119,13 @@ class FzDataUtils {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static def removeById(id){
|
||||
def del_data = new BasicDBObject()
|
||||
del_data.put("num_id", id)
|
||||
DIST_COLL.remove(del_data)
|
||||
}
|
||||
|
||||
static def saveDistrict(def m){
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("num_id",m.num_id)
|
||||
|
||||
+7
-4
@@ -92,8 +92,11 @@ class DataStandardBatch extends BaseBatch{
|
||||
|
||||
cityMap.each{city,alias->
|
||||
println "开始加载【${city}】"
|
||||
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
|
||||
districtMap.putAll(detailMap)
|
||||
// if(city == "福州"){
|
||||
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
|
||||
districtMap.putAll(detailMap)
|
||||
// }
|
||||
|
||||
}
|
||||
|
||||
while(flag){
|
||||
@@ -833,7 +836,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
}
|
||||
if(!dbo){
|
||||
if(!dbo && isStandardRoad(copyObj.address)){
|
||||
result = findAddrAliasFromMap(copyObj.city, copyObj.region, copyObj.address);
|
||||
if(result.size()==1){
|
||||
dbo = result[0];
|
||||
@@ -920,7 +923,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
def findAllNoMatchData(){
|
||||
DBObject qdata = new BasicDBObject();
|
||||
qdata.put("s_date", SHORT_DATE)
|
||||
// qdata.put("city","福州")
|
||||
qdata.put("city","福州")
|
||||
// qdata.put("source","安居客")
|
||||
// qdata.put("name",Pattern.compile("龚家宅.*"))
|
||||
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));
|
||||
|
||||
+1
-1
@@ -42,7 +42,7 @@ class ThirdDataToGatherBatch extends BaseBatch{
|
||||
}
|
||||
@Override
|
||||
void process(Object args) {
|
||||
println SHORT_DATE
|
||||
|
||||
def citys = CommonDatautils.findAllCity();
|
||||
|
||||
def weekend = DateUtils.getFirstDayOfWeek(new Date());
|
||||
|
||||
Reference in New Issue
Block a user