From 3da40ce36f9a5121ac857441f604572376f105ba Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 20 Jun 2023 11:22:46 +0800 Subject: [PATCH] update --- .../script/ccbevaluation/AddSzDealData.groovy | 24 ++++++- .../BigdataCommunityBatch.groovy | 9 ++- .../ronghe/BigdataCitypriceBatch.groovy | 6 +- .../script/chongqing/HailanTest.groovy | 66 ++++++++++++++++- .../script/commData/CommonDatautils.groovy | 71 ++++++++++++++++++- .../script/fuzhou/FzDataResultStat.groovy | 2 +- .../com/fangjia/script/fuzhou/Test1.groovy | 2 +- .../fangjia/script/gj/YouDaDataParser.groovy | 4 +- .../district/AddDistrictElevatorBatch.groovy | 2 +- .../gj/paimai/BeiKeCrawlByChrome.groovy | 28 +++++--- .../gj/paimai/LianjiaCrawlByChrome.groovy | 48 ++++++++----- .../DayUpdateDistRefreshBatch.groovy | 15 +++- .../script/jiangnanblank/FzDataUtils.groovy | 7 ++ .../thirdPriceCrawl/DataStandardBatch.groovy | 11 +-- .../ThirdDataToGatherBatch.groovy | 2 +- 15 files changed, 241 insertions(+), 56 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy index f34d40421..1c1fa2788 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy @@ -9,7 +9,7 @@ import com.mongodb.DBCollection /** * 深圳公众号成交数据录入 */ -def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳4月份二手房成交数据汇总.xls", 0, 1) +def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳5月份二手房成交数据汇总.xls", 0, 1) def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) @@ -51,7 +51,19 @@ for(def d : datas){ map.region = d['B'] def roomStr = d['E'] if(roomStr){ - map.room = roomStr.toString().toDouble().toInteger() + if(roomStr.contains("-")){ + def arrs = roomStr.split("-") + try{ + map.room = arrs[0].toString().toInteger() + }catch(Exception e){ + println "room error" + } + map.hall = arrs[1].toString().toInteger() + map.toilet = arrs[3].toString().toInteger() + }else{ + map.room = roomStr.toString().toDouble().toInteger() + } + } map.area = d['F'].toString().toDouble() @@ -59,7 +71,13 @@ for(def d : datas){ map.avg_price = (int)(d['H'].toString().toDouble()) map.sell_price = (int)(d['I'].toString().toDouble()*10000) def trade_date = d['J'] - map.trade_date = trade_date + if(trade_date){ + map.trade_date = trade_date + }else{ + def trade_date1 = DateUtils.getRandomDate("2023-05-16", "2023-05-31") + map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd") + } + map.source = "深房家族" map.create_date = new Date() map.m_date = new Date() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 17860e977..42feb5869 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -56,8 +56,8 @@ class BigdataCommunityBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"]) - def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用 -// def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用 +// def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用 + def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用 def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) rets = addDeleteDist(dists, ccbMaps, rets) rows.addAll(rets) @@ -77,9 +77,8 @@ class BigdataCommunityBatch extends BaseBatch{ FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); println "txt 生成->导Oracle" - -// delPgData() -// saveToPgSql(rows) + delPgData() + saveToPgSql(rows) // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据") // def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index ed89f59ac..9ed8300cc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -61,9 +61,9 @@ class BigdataCitypriceBatch extends BaseBatch{ def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1) def lastYearDate = DateUtils.addMonth(startDate.toString(),-12, "yyyyMMdd").toInteger(); - cityRun(citys, startDate, endDate, cityCodeMap) - regionRun(citys, startDate, endDate, cityCodeMap) -// districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap) +// cityRun(citys, startDate, endDate, cityCodeMap) +// regionRun(citys, startDate, endDate, cityCodeMap) + districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap) } def cityRun(citys, startDate, endDate, cityCodeDataMap){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy index 0d0e263e6..ff1b235f3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy @@ -40,19 +40,25 @@ class HailanTest { static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com") + static main(args) { def task = new HailanTest(); task.run(); } def run(){ - def inputfile = "D:\\数据\\微元素\\排查表.xlsx" - match(inputfile) + def inputfile = "D:\\数据\\微元素\\test.xlsx" +// addThirdPrice(inputfile) +// match(inputfile) +// tradeAvgPrice(inputfile) +// caluateTotalPrice(inputfile) // addPrice(inputfile) // addCommId("D:\\数据\\海南\\匹配上的合并数据.xlsx") // addHouse(inputfile) // stat(inputfile) // hailanStat(inputfile) + addFzFjwPrice(inputfile) } def addHouse(inputfile){ @@ -702,4 +708,60 @@ class HailanTest { } return prices; } + + def tradeAvgPrice(inputfile){ + ExcelUtils.modify(inputfile, { row -> + if(row['I'] && row['I'] == "4f4dac0a84ae1852a920ad39"){ + def dist = CommonDatautils.findDistrictById(row['I']) + if(dist){ + def tradeAvgPrice = CommonDatautils.findTrades(dist, "2023-02-01", "2023-06-01"); + row['J'] = (int)tradeAvgPrice + } + } + }) + } + + + def addThirdPrice(path){ + def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302 + def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304 + + + ExcelUtils.multiThreadingModify(path, { row -> + def cityMap = CommonDatautils.findThirdPartyPrice("福州", row['F'], row['G'], startDate, endDate) + def priceMap = cityMap.get("福州"+"##"+row['F']+"##"+row['G']) + if(priceMap){ + if(priceMap.containsKey("lianjia_esf_price")){ + row['K'] = (int)(priceMap.get("lianjia_esf_price")*0.95) + } + } + }) + } + + def caluateTotalPrice(inputfile){ + ExcelUtils.multiThreadingModify(inputfile, { row -> + if(row['J']){ + row['L'] = (int)(row['J'].toString().toDouble()*row['B'].toString().toDouble()) + }else if(row['K']){ + row['L'] = (int)(row['K'].toString().toDouble()*row['B'].toString().toDouble()) + } + }) + } + + def addFzFjwPrice(inputfile){ + ExcelUtils.modify(inputfile, { row -> + if(row['B']) { + def q = new BasicDBObject() + q.put("city", "福州") + q.put("region", row['A']) + q.put("name", row['B']) + q.put("month", 202304) + def obj = FZ_DISTRICT_STAT_COLL.findOne(q) + if (obj) { + row['D'] = obj.avg_price + } + } + + }) + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 2ec846add..bc5abebe7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1022,7 +1022,7 @@ class CommonDatautils { q.put("cat", cat); q.put("region", region) q.put("name", name); - q.put("status", 0) + q.put("visible", 0) DBObject keys = new BasicDBObject(); if(fields){ @@ -3801,5 +3801,74 @@ class CommonDatautils { } + static def findTrades(dist, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", dist.city) + q.put("region", dist.region) + + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name + } + q.put("district_name", new BasicDBObject('$in', dist.alias)) + }else{ + q.put("district_name", dist.name) + } + + q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate))) + def dbcorsor = DEAL_PRICE_COLL.find(q) + def avg_price = 0 + def total_price = 0 + def count = 0 + + for(def d : dbcorsor){ + if(d.avg_price){ + total_price = total_price + d.avg_price + count++ + } + } + if(count > 3){ + avg_price = total_price/count + } + return avg_price + } + + static def findThirdPartyPrice(def city, def region, def name, def startDate, def endDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) + def dbcursor = thirdpartColl.find(q).sort(new BasicDBObject("s_date", -1)) + def map = [:] + for(def db : dbcursor){ + def key = db.city+"##"+db.region+"##"+db.name + def dataMap = map.get(key) + if(!dataMap){ + dataMap = [:] + map.put(key, dataMap) + } + + if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){ + dataMap.put("anjuke_esf_price", db.anjuke_esf_price) + } + if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){ + dataMap.put("anjuke_esf_price", db.get("58tc_esf_price")) + } + + if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){ + dataMap.put("lianjia_esf_price", db.lianjia_esf_price) + } + if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){ + dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price) + } + + if(!dataMap.containsKey("lianjia_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){ + dataMap.put("lianjia_esf_price", db.beike_esf_price) + } + } + return map + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy index 2da506d11..917ddfc5e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy @@ -25,7 +25,7 @@ class FzDataResultStat { static main(args) { def task = new FzDataResultStat() task.priceMarkupRun(); -// task.regionPriceTrendRun() + task.regionPriceTrendRun() //// task.margeExcel() // task.saveCsvData() } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index a3be166aa..63211976b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202304.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202305.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index 16cae9383..4de90b96b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -48,11 +48,11 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2023-04-01" + static def statDate = "2023-05-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo -// task.esfParser("D:\\数据\\友达\\2023年5月\\查询1.DBF");//友达二手房存入mongo +// task.esfParser("D:\\数据\\友达\\2023年6月\\查询1.DBF");//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es task.esfhouseToSaveEs();//二手房保存到es // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/AddDistrictElevatorBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/AddDistrictElevatorBatch.groovy index 10efae15e..cc9c87682 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/AddDistrictElevatorBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/AddDistrictElevatorBatch.groovy @@ -65,7 +65,7 @@ class AddDistrictElevatorBatch extends BaseBatch{ } def newHouses = analyElevator(haveHouses, dist.estate_type2) if(newHouses && newHouses.size()>=3 && newHouses.size()>=haveHouses.size()*0.6){ - println totalCount.get()+" [${dist.estate_type2}] "+dist.name +" 原:"+houses.size()+" 有值:"+haveHouses.size()+" 10层以上:"+newHouses.size() + println totalCount.get()+" [${dist.estate_type2}] "+dist.name +" 原:"+houses.size()+" 有值:"+haveHouses.size()+" 筛选样本量:"+newHouses.size() BasicDBObject udata = new BasicDBObject() if(dist.estate_type2 == "别墅"){ udata.put("is_elevator", false) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 1068d1f84..b190a324a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -289,20 +289,26 @@ public class BeiKeCrawlByChrome { } detail.source = "贝壳找房" - def liftStr = JQueryUtils.selectText("li:contains(梯户比例)",detailDoc).replace("梯户比例","") + def detailDiv1 = JQueryUtils.select(".introContent .base", detailDoc) + def detailDiv2 = JQueryUtils.select(".introContent .transaction", detailDoc) + + def liftStr = detailDiv1.select("li:contains(梯户比例)").text().replace("梯户比例","") if(liftStr){ detail.lift = DataFormatUtils.parseLift(liftStr) detail.lift_household = DataFormatUtils.parseHousehold(liftStr) } - detail.trade_property = JQueryUtils.selectText("li:contains(交易权属)", detailDoc).replace("交易权属","") - detail.room_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(户型结构)", detailDoc).replace("户型结构","")) - detail.property_ownership = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房权所属)", detailDoc).replace("房权所属","")) - detail.housing_age = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房屋年限)", detailDoc).replace("房屋年限","")) - detail.elevator = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(配备电梯)", detailDoc).replace("配备电梯","")) - detail.build_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑结构)", detailDoc).replace("建筑结构","")) - detail.build_year = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建成年代)", detailDoc).replace("建成年代","")) - detail.build_type = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑类型)", detailDoc).replace("建筑类型","")) - def floorStr = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(所在楼层)", detailDoc).replace("㎡","")) + detail.room_structure = DataFormatUtils.getValue(detailDiv1.select("li:contains(户型结构)").text().replace("户型结构","")) + detail.elevator = DataFormatUtils.getValue(detailDiv1.select("li:contains(配备电梯)").text().replace("配备电梯","")) + detail.build_structure = DataFormatUtils.getValue(detailDiv1.select("li:contains(建筑结构)").text().replace("建筑结构","")) + detail.build_year = DataFormatUtils.getValue(detailDiv1.select("li:contains(建成年代)").text().replace("建成年代","")) + detail.build_type = DataFormatUtils.getValue(detailDiv1.select("li:contains(建筑类型)").text().replace("建筑类型","")) + def floorStr = DataFormatUtils.getValue(detailDiv1.select("li:contains(所在楼层)").text().replace("㎡","")) + + detail.trade_property = detailDiv2.select("li:contains(交易权属)").text().replace("交易权属","") + detail.property_ownership = DataFormatUtils.getValue(detailDiv2.select("li:contains(房权所属)").text().replace("房权所属","")) + detail.house_use = DataFormatUtils.getValue(detailDiv2.select("li:contains(房屋用途)").text().replace("房屋用途","")) + detail.house_use = DataFormatUtils.getValue(detailDiv2.select("li:contains(房屋用途)").text().replace("房屋用途","")) + detail.housing_age = DataFormatUtils.getValue(detailDiv2.select("li:contains(房屋年限)").text().replace("房屋年限","")) if(detail.total_price && detail.avg_price){ detail.url = detailUrl @@ -483,7 +489,7 @@ public class BeiKeCrawlByChrome { elemPassword.sendKeys("12qwaszx"); btn.submit(); // 点击行为; - Thread.sleep(20000) + Thread.sleep(10000) }catch(Exception e){ Thread.sleep(15000) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 0c82ca47e..9ab7eacb3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -297,16 +297,23 @@ public class LianjiaCrawlByChrome { def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd") - if(dealDate.getTime()>=endDate.getTime()){ - detailUrls << durl - def dbo = findTrade(durl , "链家在线") - if (!dbo) { - parse(city, e, regionText, driver) + if(dealDate){ + if(dealDate.getTime()>=endDate.getTime()){ + detailUrls << durl + println "1..."+new Date() + def dbo = findTrade(durl , "链家在线") + println "2..."+new Date() + if (!dbo) { + parse(city, e, regionText, driver) + } + }else{ + stop = true; + break; } }else{ - stop = true; - break; + println url +" 成交时间有误!" } + } if(page>=100){ stop = true; @@ -326,8 +333,9 @@ public class LianjiaCrawlByChrome { if(price){ def detailUrl = e.select(".title a").attr("href") driver.get(detailUrl) + println "3..."+new Date() def detailDoc = checkDocument(driver, detailUrl); - + println "4..."+new Date() def title = e.select(".title a").text() if (title) { def detail = new BasicDBObject() @@ -386,20 +394,22 @@ public class LianjiaCrawlByChrome { detail.source = "链家在线" detail.url = detailUrl - def liftStr = JQueryUtils.selectText("li:contains(梯户比例)",detailDoc).replace("梯户比例","") + def detailDiv = JQueryUtils.select(".introContent", detailDoc) + + def liftStr = detailDiv.select("li:contains(梯户比例)").text().replace("梯户比例","") if(liftStr){ detail.lift = DataFormatUtils.parseLift(liftStr) detail.lift_household = DataFormatUtils.parseHousehold(liftStr) } - detail.trade_property = JQueryUtils.selectText("li:contains(交易权属)", detailDoc).replace("交易权属","") - detail.room_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(户型结构)", detailDoc).replace("户型结构","")) - detail.property_ownership = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房权所属)", detailDoc).replace("房权所属","")) - detail.housing_age = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房屋年限)", detailDoc).replace("房屋年限","")) - detail.elevator = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(配备电梯)", detailDoc).replace("配备电梯","")) - detail.build_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑结构)", detailDoc).replace("建筑结构","")) - detail.build_year = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建成年代)", detailDoc).replace("建成年代","")) - detail.build_type = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑类型)", detailDoc).replace("建筑类型","")) - def floorStr = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(所在楼层)", detailDoc).replace("所在楼层","")) + detail.trade_property = detailDiv.select("li:contains(交易权属)").text().replace("交易权属","") + detail.room_structure = DataFormatUtils.getValue( detailDiv.select("li:contains(户型结构)").text().replace("户型结构","")) + detail.property_ownership = DataFormatUtils.getValue( detailDiv.select("li:contains(房权所属)").text().replace("房权所属","")) + detail.housing_age = DataFormatUtils.getValue( detailDiv.select("li:contains(房屋年限)").text().replace("房屋年限","")) + detail.elevator = DataFormatUtils.getValue( detailDiv.select("li:contains(配备电梯)").text().replace("配备电梯","")) + detail.build_structure = DataFormatUtils.getValue( detailDiv.select("li:contains(建筑结构)").text().replace("建筑结构","")) + detail.build_year = DataFormatUtils.getValue( detailDiv.select("li:contains(建成年代)").text().replace("建成年代","")) + detail.build_type = DataFormatUtils.getValue( detailDiv.select("li:contains(建筑类型)").text().replace("建筑类型","")) + def floorStr = DataFormatUtils.getValue( detailDiv.select("li:contains(所在楼层)").text().replace("所在楼层","")) if(detail.total_price && detail.avg_price){ def houseInfo = e.select(".houseInfo").text() @@ -417,7 +427,9 @@ public class LianjiaCrawlByChrome { detail.create_date = new Date() detail.m_date = new Date() println detail + println "5..."+new Date() DEAL_PRICE_COLL.save(detail) + println "6..."+new Date() } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/DayUpdateDistRefreshBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/DayUpdateDistRefreshBatch.groovy index bb19fe69d..72df7724e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/DayUpdateDistRefreshBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/DayUpdateDistRefreshBatch.groovy @@ -28,9 +28,18 @@ class DayUpdateDistRefreshBatch extends BaseBatch{ if(dists){ for(def d : dists){ - def distObj = CommonDatautils.findDistrictByParams(d.city, d.region, d.name, "district", fields) + def distObj = null + if(d.comm_id){ + distObj = CommonDatautils.findDistrictById(d.comm_id) + }else{ + distObj = CommonDatautils.findDistrictByParams(d.city, d.region, d.name, "district", fields) + } if(distObj && distObj._id){ - FzDataUtils.converDistrict(distObj, null, null); + if(d.type && d.type==1){ + FzDataUtils.removeById(d.comm_id) + }else{ + FzDataUtils.converDistrict(distObj, null, null); + } println new Date().toString()+" 【处理完成】 "+d.city +" "+d.region+" "+d.name updateTaskStatus(d.id, 2); } @@ -57,7 +66,7 @@ class DayUpdateDistRefreshBatch extends BaseBatch{ def q = new BasicDBObject() q.put("status", 5);//未处理 TASK_COLL.find(q).each {d-> - rets << ["id":d._id.toString(),"city":d.city, "region":d.region, "name":d.name] + rets << ["id":d._id.toString(),"city":d.city, "region":d.region, "name":d.name, "type":d.type, "comm_id":d.comm_id] } return rets; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/FzDataUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/FzDataUtils.groovy index bf6e2c177..51e58e28e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/FzDataUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/FzDataUtils.groovy @@ -119,6 +119,13 @@ class FzDataUtils { } } + + static def removeById(id){ + def del_data = new BasicDBObject() + del_data.put("num_id", id) + DIST_COLL.remove(del_data) + } + static def saveDistrict(def m){ DBObject q = new BasicDBObject(); q.put("num_id",m.num_id) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index b5f86bf1b..1c287e355 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -92,8 +92,11 @@ class DataStandardBatch extends BaseBatch{ cityMap.each{city,alias-> println "开始加载【${city}】" - def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"]) - districtMap.putAll(detailMap) +// if(city == "福州"){ + def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"]) + districtMap.putAll(detailMap) +// } + } while(flag){ @@ -833,7 +836,7 @@ class DataStandardBatch extends BaseBatch{ } } } - if(!dbo){ + if(!dbo && isStandardRoad(copyObj.address)){ result = findAddrAliasFromMap(copyObj.city, copyObj.region, copyObj.address); if(result.size()==1){ dbo = result[0]; @@ -920,7 +923,7 @@ class DataStandardBatch extends BaseBatch{ def findAllNoMatchData(){ DBObject qdata = new BasicDBObject(); qdata.put("s_date", SHORT_DATE) -// qdata.put("city","福州") + qdata.put("city","福州") // qdata.put("source","安居客") // qdata.put("name",Pattern.compile("龚家宅.*")) qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false)); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy index 320e65b2f..4fe5dd4a7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy @@ -42,7 +42,7 @@ class ThirdDataToGatherBatch extends BaseBatch{ } @Override void process(Object args) { - println SHORT_DATE + def citys = CommonDatautils.findAllCity(); def weekend = DateUtils.getFirstDayOfWeek(new Date());