From e441a10add801c62b94ab3255ab9622ccb82dd6c Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 12 May 2023 18:36:11 +0800 Subject: [PATCH] update --- .../com/fangjia/script/fuzhou/Test1.groovy | 2 +- .../gj/district/DistrictFieldAddBatch.groovy | 5 ++- .../gj/paimai/BeiKeCrawlByChrome.groovy | 44 ++++++++++++++----- .../gj/paimai/LianjiaCrawlByChrome.groovy | 31 ++++++++++--- 4 files changed, 63 insertions(+), 19 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index e26dc05dd..a3be166aa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202303.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202304.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy index f3a47806d..2806a48f1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy @@ -88,7 +88,7 @@ class DistrictFieldAddBatch extends BaseBatch{ for(def c : citys){ def totalCount = CommonDatautils.findAllDistrictCount("district", c) if(totalCount){ - rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets) +// rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets) analysRets = addFieldFromMatch(c, totalCount, analysRets) } } @@ -175,6 +175,9 @@ class DistrictFieldAddBatch extends BaseBatch{ def fields = fieldsMap.keySet() as List def fieldValueMap = [:] for(def rhDist : rhRets){ + if(rhDist.commid && rhDist.commid!="null"){ + continue + } def dist = CommonDatautils.findDistrictFieldById(rhDist.commid, fields) if(!dist){ println c+" "+rhDist.commid+" 不存在" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 916db9961..ca92bb2c3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -69,7 +69,7 @@ public class BeiKeCrawlByChrome { // // } // citys = citys[(start+1)..(citys.size()-1)] - getLogin(driver) + driver = getLogin() crawlData(driver, citys) @@ -108,15 +108,32 @@ public class BeiKeCrawlByChrome { } } + def checkDocument(driver, url){ + def s = driver.getPageSource() + def flag = true + while(flag){ + if(s && s.contains("无法访问此网站")){ + driver = getLogin() + driver.get(url) + Thread.sleep(5000) + s = driver.getPageSource() + }else{ + flag = false + } + } + return Jsoup.parse(s); + } + def crawlData(driver, citys){ def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) for(def m : citys){ def city = m[0] def url = m[1] - driver.get(url+"/chengjiao/") + def newUrl = url+"/chengjiao/" + driver.get(newUrl) Thread.sleep(10000) - def doc = Jsoup.parse(driver.getPageSource()); + def doc = checkDocument(driver, newUrl); def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc) if(regions){ for(def r : regions){ @@ -220,6 +237,9 @@ public class BeiKeCrawlByChrome { } def roomStr = arrs[1] + if(roomStr && !roomStr.contains("室")){ + roomStr = arrs[2] + } if (roomStr) { if (roomStr.indexOf("室") > 0) { roomStr = roomStr.substring(0, roomStr.indexOf("室")) @@ -235,16 +255,16 @@ public class BeiKeCrawlByChrome { } } - def areaStr = arrs[2].replace("平米", "") + def areaStr = arrs[2] + if(areaStr && !areaStr.contains("平米")){ + if(arrs.size()>=4){ + areaStr = arrs[3] + } + } + areaStr = areaStr.replace("平米","") if (areaStr && areaStr.isNumber()) { detail.area = areaStr.toDouble() } - if(!detail.area){ - areaStr = JQueryUtils.selectText("li:contains(建筑面积)", doc).replace("㎡", "") - if (areaStr && areaStr.isNumber()) { - detail.area = areaStr.toDouble() - } - } def price = e.select(".totalPrice .number").text() if (price && price.isNumber()) { @@ -418,7 +438,8 @@ public class BeiKeCrawlByChrome { return list } - def getLogin(driver){ + def getLogin(){ + WebDriver driver = new ChromeDriver() driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.ke.com%252Fchengjiao%252F"); WebDriverWait wait = new WebDriverWait(driver,20) Thread.sleep(10000) @@ -438,6 +459,7 @@ public class BeiKeCrawlByChrome { btn.submit(); // 点击行为; Thread.sleep(10000) + return driver } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 6c01e6c11..bc5448c6c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -122,7 +122,7 @@ public class LianjiaCrawlByChrome { WebDriver driver = new ChromeDriver() def cityMap = getAllCityUrls(driver) // cityMap = cityMap.reverse() - getLogin(driver) + driver = getLogin() crawlData(driver, cityMap) @@ -191,10 +191,12 @@ public class LianjiaCrawlByChrome { for(def ret : cityList){ def city = ret[0] def cityUrl = ret[1] + def newUrl = cityUrl+"chengjiao/" + driver.get(newUrl) //def detailUrls = getPageUrls(driver, cityUrl) - driver.get(cityUrl+"chengjiao/") Thread.sleep(10000) - def doc = Jsoup.parse(driver.getPageSource()); + def doc = checkDocument(driver, newUrl); + def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc) if(regions){ for(def r : regions){ @@ -240,6 +242,21 @@ public class LianjiaCrawlByChrome { return detailUrls } + def checkDocument(driver, url){ + def s = driver.getPageSource() + def flag = true + while(flag){ + if(s && s.contains("无法访问此网站")){ + driver = getLogin() + driver.get(url) + Thread.sleep(5000) + s = driver.getPageSource() + }else{ + flag = false + } + } + return Jsoup.parse(s); + } def getPageDeail(driver, city, cityUrl, regionText){ def stop = false @@ -278,7 +295,7 @@ public class LianjiaCrawlByChrome { def parse(city,e, regionText){ try { - def title = e.select(".title a").text() + def title = e.select(".title a").text() if (title) { def detail = new BasicDBObject() def arrs = title.split(" ") @@ -318,7 +335,7 @@ public class LianjiaCrawlByChrome { } } - def price = e.select(".totalPrice .number").text() + def price = e.select(".totalPrice .number").text() if (price && price.isNumber()) { detail.total_price = (int)(price.toDouble() * 10000) } @@ -490,7 +507,8 @@ public class LianjiaCrawlByChrome { return cityList } - def getLogin(driver){ + def getLogin(){ + WebDriver driver = new ChromeDriver() driver.get("https://clogin.lianjia.com/login?service=https%3A%2F%2Fwww.lianjia.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.lianjia.com%252Fchengjiao%252F"); WebDriverWait wait = new WebDriverWait(driver,20) Thread.sleep(10000) @@ -510,6 +528,7 @@ public class LianjiaCrawlByChrome { btn.submit(); // 点击行为; Thread.sleep(20000) + return driver } }