+2
-2
@@ -26,7 +26,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
void process(Object args) {
|
||||
def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
|
||||
//按天导(每天一个文件夹)
|
||||
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"))
|
||||
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"))
|
||||
|
||||
if(curDate.toString().endsWith("10")){
|
||||
exportBigdataCommunityGJRH()
|
||||
@@ -471,7 +471,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
def allRows = []
|
||||
if(curDate.toString().endsWith("01")){
|
||||
if(curDate.toString().endsWith("28")){
|
||||
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
|
||||
+2
-2
@@ -34,7 +34,7 @@ public class BeiKeCrawlByChrome {
|
||||
|
||||
static main(args) {
|
||||
def task = new BeiKeCrawlByChrome();
|
||||
task.run()
|
||||
// task.run()
|
||||
// task.run1()
|
||||
task.addDealRegion();
|
||||
}
|
||||
@@ -342,7 +342,7 @@ public class BeiKeCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
def href = e.attr("href")
|
||||
if(e.text().contains("南京")){
|
||||
if(!e.text().contains("南京")){
|
||||
cityMap.put(e.text(), "https:"+href)
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -34,7 +34,7 @@ public class LianjiaCrawlByChrome {
|
||||
|
||||
static main(args) {
|
||||
def task = new LianjiaCrawlByChrome();
|
||||
// task.run()
|
||||
task.run()
|
||||
// task.run1()
|
||||
task.addDealRegion();
|
||||
// task.standardRegion()
|
||||
|
||||
+261
@@ -0,0 +1,261 @@
|
||||
package com.fangjia.script.loudong
|
||||
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.jsoup.Jsoup
|
||||
import org.openqa.selenium.By
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.WebElement
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
import org.openqa.selenium.interactions.Actions
|
||||
import org.openqa.selenium.support.ui.ExpectedCondition
|
||||
import org.openqa.selenium.support.ui.ExpectedConditions
|
||||
import org.openqa.selenium.support.ui.WebDriverWait
|
||||
|
||||
class HouseBookChromeCrawl {
|
||||
|
||||
static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong", "192.168.0.235", 27018)
|
||||
static def city = "重庆"
|
||||
static main(args) {
|
||||
def task = new HouseBookChromeCrawl();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def listPageUrl = getIndexPage(driver)
|
||||
def keywords = getListPage(driver,"http://www.house-book.com.cn/"+listPageUrl)
|
||||
keywords.remove("城南家园")
|
||||
keywords.remove("康居西城")
|
||||
|
||||
println city +"共 "+keywords.size()+" 个小区"
|
||||
getDistrictIndex(driver, keywords)
|
||||
|
||||
}
|
||||
|
||||
def getListPage(driver, url){
|
||||
driver.get(url)
|
||||
|
||||
def cityDoc = Jsoup.parse(driver.getPageSource())
|
||||
def total = JQueryUtils.selectText(".result-count-number:eq(0)", cityDoc)
|
||||
if(total){
|
||||
def page = getTotalPage(total, 6)
|
||||
def keywords = getAllDist(driver, url, page);
|
||||
return keywords
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
def getDistrictIndex(driver, keywords){
|
||||
for(def word : keywords){
|
||||
def code = URLEncoder.encode(URLEncoder.encode(word, "utf-8"),"utf-8")
|
||||
goToDetailIndex(driver, code)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def getAllDist(driver, url, page){
|
||||
def keywords = []
|
||||
for(def i=1;i<=page;i++){
|
||||
driver.get(url+"?selectPages="+i)
|
||||
def doc = Jsoup.parse(driver.getPageSource())
|
||||
def eles = JQueryUtils.select(".title-area .title-text", doc)
|
||||
for(def e : eles){
|
||||
def text = e.text()
|
||||
if(text=="富州北郡"){
|
||||
println i
|
||||
}
|
||||
keywords << text
|
||||
}
|
||||
}
|
||||
return keywords
|
||||
}
|
||||
|
||||
def getTotalPage(def total, def pageSize){
|
||||
def mode = total.toInteger()/pageSize
|
||||
if(mode > mode.toInteger()){
|
||||
mode = mode.toInteger() +1
|
||||
}
|
||||
return mode;
|
||||
}
|
||||
|
||||
|
||||
def goToDetailIndex(driver, code){
|
||||
def url = "http://www.house-book.com.cn/buildAction!pnameHome?pname="+code+"&price=&totalPrice=&bsqar=&htype=&btype=&sellKind="
|
||||
driver.get(url)
|
||||
|
||||
WebDriverWait wait = new WebDriverWait(driver,20)
|
||||
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("main"))));
|
||||
def docment = Jsoup.parse(driver.getPageSource());
|
||||
def dist_name = JQueryUtils.select(".info-pname", docment).text()
|
||||
def addInfo = JQueryUtils.select(".addr-info", docment).text()
|
||||
def region = null
|
||||
if(addInfo.contains(":") && addInfo.contains("-")){
|
||||
region = addInfo.substring(addInfo.indexOf(":")+1, addInfo.indexOf("-"))
|
||||
}
|
||||
|
||||
def address = JQueryUtils.select(".addr-info span", docment).text()
|
||||
|
||||
// wait.until(new ExpectedCondition<WebElement>(){
|
||||
// @Override
|
||||
// public WebElement apply(WebDriver d){
|
||||
// return d.findElement(By.linkText("[更多]"))
|
||||
// }
|
||||
// }).click()
|
||||
driver.findElement(By.linkText("[更多]")).click()
|
||||
// Thread.sleep(10)
|
||||
// wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.id("shouUl"))));
|
||||
def lis = checkElement(driver, "#shouUl .dongli");
|
||||
|
||||
def datas = []
|
||||
if(lis){
|
||||
for(def li : lis){
|
||||
def id = li.attr("id")
|
||||
def houseNum = li.select(".title-span").text()
|
||||
def map = [:]
|
||||
map.id = id
|
||||
map.city = city
|
||||
map.region = region
|
||||
map.district_name = dist_name
|
||||
map.address = address
|
||||
map.house_num = houseNum
|
||||
map.num = li.select(".dong_ul2 li:contains(许可证号:)").text().replace("许可证号:","")
|
||||
map.type = li.select(".dong_ul2 .title-span-hbType").text()
|
||||
def floor_heigth = li.select(".dong_ul2 label:contains(总楼层:)").text().replace("总楼层:","").replace("层","")
|
||||
if(floor_heigth){
|
||||
map.floor_heigth = floor_heigth
|
||||
}
|
||||
def room_count = li.select(".dong_ul2 label:contains(总户数:)").text()
|
||||
if(room_count){
|
||||
room_count = room_count.substring(room_count.indexOf("总户数:")+4, room_count.lastIndexOf("户"))
|
||||
if(room_count){
|
||||
map.household_count = room_count
|
||||
}
|
||||
}
|
||||
datas << map
|
||||
}
|
||||
}
|
||||
|
||||
println dist_name +" 共 "+datas.size()+" 栋"
|
||||
for(def m : datas){
|
||||
try{
|
||||
driver.findElement(By.id(m.id)).click()
|
||||
|
||||
def rows = parseDetail(driver)
|
||||
println "栋divId: "+m.id +" 室:"+rows.size()
|
||||
saveDate(m, rows)
|
||||
}catch(Exception e){
|
||||
println dist_name+" 栋divId: "+m.id+" 找不到相应的div element"
|
||||
}
|
||||
|
||||
// driver.navigate().back();
|
||||
}
|
||||
}
|
||||
|
||||
def checkElement(driver, className){
|
||||
def loop = 0
|
||||
def stop = false
|
||||
def lis = null
|
||||
while (!stop){
|
||||
loop++
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
lis = JQueryUtils.select(className, doc)
|
||||
if(lis || loop>=10){
|
||||
stop = true;
|
||||
}else{
|
||||
Thread.sleep(5000)
|
||||
}
|
||||
}
|
||||
return lis
|
||||
|
||||
}
|
||||
|
||||
def parseDetail(driver){
|
||||
def trs = checkElement(driver, ".rooms tr");
|
||||
def rows = []
|
||||
for(def tr : trs){
|
||||
def tds = tr.select("td")
|
||||
if(tds){
|
||||
def floor = null
|
||||
for(def j=0;j<tds.size();j++){
|
||||
if(j == 0){
|
||||
floor = tds[0].text().replace("第","").replace("层","")
|
||||
}else {
|
||||
def td = tds[j]
|
||||
def detail = [:]
|
||||
def room_num = td.select("b").text()
|
||||
if (room_num) {
|
||||
detail.room_num = room_num
|
||||
}
|
||||
def area = td.select("span:contains(面积:)").text().replace("面积:", "").replace("㎡", "")
|
||||
if (area) {
|
||||
detail.area = area.toDouble()
|
||||
}
|
||||
def avg_price = td.select("span:contains(单价:)").text().replace("单价:", "").replace("元/㎡", "")
|
||||
if (avg_price) {
|
||||
detail.avg_price = avg_price.toInteger()
|
||||
}
|
||||
|
||||
def house_type = td.select("span:contains(房屋类型:)").text().replace("房屋类型:", "")
|
||||
if(house_type){
|
||||
detail.estate_type = house_type
|
||||
}
|
||||
if(detail){
|
||||
=-09ql.floor = floor
|
||||
rows << detail
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return rows;
|
||||
}+*-*
|
||||
|
||||
|
||||
def getIndexPage(driver){
|
||||
driver.get("http://www.house-book.com.cn/mainAction/city/%25E9%2587%258D%25E5%25BA%2586%25E5%25B8%2582.html")
|
||||
Thread.sleep(8000)
|
||||
def cityDoc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles = JQueryUtils.select("a:contains(新 房)", cityDoc)
|
||||
if(cityEles){
|
||||
def listPageUrl = cityEles[0].attr("href")
|
||||
return listPageUrl
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def saveDate(m, rows){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", m.region)
|
||||
q.put("district_name", m.district_name)
|
||||
def datas = []
|
||||
for(def map : rows){
|
||||
q.put("house_num", m.house_num)
|
||||
q.put("room_num", map.room_num)
|
||||
q.put("floor_heigth" , m.floor_heigth)
|
||||
q.put("floor", map.floor)
|
||||
q.put("area", map.area)
|
||||
def obj = loudongConn.findOne(q)
|
||||
if(!obj){
|
||||
def data = new BasicDBObject()
|
||||
m.each{k,v->
|
||||
data.put(k,v)
|
||||
}
|
||||
map.each{k,v->
|
||||
data.put(k,v)
|
||||
}
|
||||
data.put("c_date", new Date())
|
||||
datas << data
|
||||
}
|
||||
}
|
||||
if(datas){
|
||||
loudongConn.insert(datas)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user