From d816d79e25770208f8076c9eeb15d8b4d1ea0bcd Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 27 Mar 2022 12:23:26 +0800 Subject: [PATCH] =?UTF-8?q?=E6=88=90=E4=BA=A41->28?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ronghe/ExportDistrictRongHe.groovy | 4 +- .../gj/paimai/BeiKeCrawlByChrome.groovy | 4 +- .../gj/paimai/LianjiaCrawlByChrome.groovy | 2 +- .../loudong/HouseBookChromeCrawl.groovy | 261 ++++++++++++++++++ 4 files changed, 266 insertions(+), 5 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/HouseBookChromeCrawl.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index fff8d13cf..d81974c8b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -26,7 +26,7 @@ class ExportDistrictRongHe extends BaseBatch{ void process(Object args) { def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") //按天导(每天一个文件夹) - exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd")) + exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd")) if(curDate.toString().endsWith("10")){ exportBigdataCommunityGJRH() @@ -471,7 +471,7 @@ class ExportDistrictRongHe extends BaseBatch{ } def allRows = [] - if(curDate.toString().endsWith("01")){ + if(curDate.toString().endsWith("28")){ def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and sqlInstance.eachRow(sql,{r-> def data = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index aa3cb1328..6d75b18bd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -34,7 +34,7 @@ public class BeiKeCrawlByChrome { static main(args) { def task = new BeiKeCrawlByChrome(); - task.run() +// task.run() // task.run1() task.addDealRegion(); } @@ -342,7 +342,7 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = e.attr("href") - if(e.text().contains("南京")){ + if(!e.text().contains("南京")){ cityMap.put(e.text(), "https:"+href) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index fca2a5eff..76d63a6a5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -34,7 +34,7 @@ public class LianjiaCrawlByChrome { static main(args) { def task = new LianjiaCrawlByChrome(); -// task.run() + task.run() // task.run1() task.addDealRegion(); // task.standardRegion() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/HouseBookChromeCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/HouseBookChromeCrawl.groovy new file mode 100644 index 000000000..31ab75ba6 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/HouseBookChromeCrawl.groovy @@ -0,0 +1,261 @@ +package com.fangjia.script.loudong + +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.jsoup.Jsoup +import org.openqa.selenium.By +import org.openqa.selenium.WebDriver +import org.openqa.selenium.WebElement +import org.openqa.selenium.chrome.ChromeDriver +import org.openqa.selenium.interactions.Actions +import org.openqa.selenium.support.ui.ExpectedCondition +import org.openqa.selenium.support.ui.ExpectedConditions +import org.openqa.selenium.support.ui.WebDriverWait + +class HouseBookChromeCrawl { + + static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong", "192.168.0.235", 27018) + static def city = "重庆" + static main(args) { + def task = new HouseBookChromeCrawl(); + task.run() + } + + def run(){ + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + + WebDriver driver = new ChromeDriver() + def listPageUrl = getIndexPage(driver) + def keywords = getListPage(driver,"http://www.house-book.com.cn/"+listPageUrl) + keywords.remove("城南家园") + keywords.remove("康居西城") + + println city +"共 "+keywords.size()+" 个小区" + getDistrictIndex(driver, keywords) + + } + + def getListPage(driver, url){ + driver.get(url) + + def cityDoc = Jsoup.parse(driver.getPageSource()) + def total = JQueryUtils.selectText(".result-count-number:eq(0)", cityDoc) + if(total){ + def page = getTotalPage(total, 6) + def keywords = getAllDist(driver, url, page); + return keywords + } + return null; + } + + def getDistrictIndex(driver, keywords){ + for(def word : keywords){ + def code = URLEncoder.encode(URLEncoder.encode(word, "utf-8"),"utf-8") + goToDetailIndex(driver, code) + + } + } + + def getAllDist(driver, url, page){ + def keywords = [] + for(def i=1;i<=page;i++){ + driver.get(url+"?selectPages="+i) + def doc = Jsoup.parse(driver.getPageSource()) + def eles = JQueryUtils.select(".title-area .title-text", doc) + for(def e : eles){ + def text = e.text() + if(text=="富州北郡"){ + println i + } + keywords << text + } + } + return keywords + } + + def getTotalPage(def total, def pageSize){ + def mode = total.toInteger()/pageSize + if(mode > mode.toInteger()){ + mode = mode.toInteger() +1 + } + return mode; + } + + + def goToDetailIndex(driver, code){ + def url = "http://www.house-book.com.cn/buildAction!pnameHome?pname="+code+"&price=&totalPrice=&bsqar=&htype=&btype=&sellKind=" + driver.get(url) + + WebDriverWait wait = new WebDriverWait(driver,20) + wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("main")))); + def docment = Jsoup.parse(driver.getPageSource()); + def dist_name = JQueryUtils.select(".info-pname", docment).text() + def addInfo = JQueryUtils.select(".addr-info", docment).text() + def region = null + if(addInfo.contains(":") && addInfo.contains("-")){ + region = addInfo.substring(addInfo.indexOf(":")+1, addInfo.indexOf("-")) + } + + def address = JQueryUtils.select(".addr-info span", docment).text() + +// wait.until(new ExpectedCondition(){ +// @Override +// public WebElement apply(WebDriver d){ +// return d.findElement(By.linkText("[更多]")) +// } +// }).click() + driver.findElement(By.linkText("[更多]")).click() +// Thread.sleep(10) +// wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.id("shouUl")))); + def lis = checkElement(driver, "#shouUl .dongli"); + + def datas = [] + if(lis){ + for(def li : lis){ + def id = li.attr("id") + def houseNum = li.select(".title-span").text() + def map = [:] + map.id = id + map.city = city + map.region = region + map.district_name = dist_name + map.address = address + map.house_num = houseNum + map.num = li.select(".dong_ul2 li:contains(许可证号:)").text().replace("许可证号:","") + map.type = li.select(".dong_ul2 .title-span-hbType").text() + def floor_heigth = li.select(".dong_ul2 label:contains(总楼层:)").text().replace("总楼层:","").replace("层","") + if(floor_heigth){ + map.floor_heigth = floor_heigth + } + def room_count = li.select(".dong_ul2 label:contains(总户数:)").text() + if(room_count){ + room_count = room_count.substring(room_count.indexOf("总户数:")+4, room_count.lastIndexOf("户")) + if(room_count){ + map.household_count = room_count + } + } + datas << map + } + } + + println dist_name +" 共 "+datas.size()+" 栋" + for(def m : datas){ + try{ + driver.findElement(By.id(m.id)).click() + + def rows = parseDetail(driver) + println "栋divId: "+m.id +" 室:"+rows.size() + saveDate(m, rows) + }catch(Exception e){ + println dist_name+" 栋divId: "+m.id+" 找不到相应的div element" + } + + // driver.navigate().back(); + } + } + + def checkElement(driver, className){ + def loop = 0 + def stop = false + def lis = null + while (!stop){ + loop++ + def doc = Jsoup.parse(driver.getPageSource()); + lis = JQueryUtils.select(className, doc) + if(lis || loop>=10){ + stop = true; + }else{ + Thread.sleep(5000) + } + } + return lis + + } + + def parseDetail(driver){ + def trs = checkElement(driver, ".rooms tr"); + def rows = [] + for(def tr : trs){ + def tds = tr.select("td") + if(tds){ + def floor = null + for(def j=0;j + data.put(k,v) + } + map.each{k,v-> + data.put(k,v) + } + data.put("c_date", new Date()) + datas << data + } + } + if(datas){ + loudongConn.insert(datas) + } + } +}