成交1->28

This commit is contained in:
2022-03-27 12:23:26 +08:00
parent 26cb0cec5d
commit d816d79e25
4 changed files with 266 additions and 5 deletions
@@ -26,7 +26,7 @@ class ExportDistrictRongHe extends BaseBatch{
void process(Object args) {
def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
//按天导(每天一个文件夹)
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"))
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"))
if(curDate.toString().endsWith("10")){
exportBigdataCommunityGJRH()
@@ -471,7 +471,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def allRows = []
if(curDate.toString().endsWith("01")){
if(curDate.toString().endsWith("28")){
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and
sqlInstance.eachRow(sql,{r->
def data = []
@@ -34,7 +34,7 @@ public class BeiKeCrawlByChrome {
static main(args) {
def task = new BeiKeCrawlByChrome();
task.run()
// task.run()
// task.run1()
task.addDealRegion();
}
@@ -342,7 +342,7 @@ public class BeiKeCrawlByChrome {
if(cityEles){
for(def e : cityEles){
def href = e.attr("href")
if(e.text().contains("南京")){
if(!e.text().contains("南京")){
cityMap.put(e.text(), "https:"+href)
}
}
@@ -34,7 +34,7 @@ public class LianjiaCrawlByChrome {
static main(args) {
def task = new LianjiaCrawlByChrome();
// task.run()
task.run()
// task.run1()
task.addDealRegion();
// task.standardRegion()
@@ -0,0 +1,261 @@
package com.fangjia.script.loudong
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.jsoup.Jsoup
import org.openqa.selenium.By
import org.openqa.selenium.WebDriver
import org.openqa.selenium.WebElement
import org.openqa.selenium.chrome.ChromeDriver
import org.openqa.selenium.interactions.Actions
import org.openqa.selenium.support.ui.ExpectedCondition
import org.openqa.selenium.support.ui.ExpectedConditions
import org.openqa.selenium.support.ui.WebDriverWait
class HouseBookChromeCrawl {
static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong", "192.168.0.235", 27018)
static def city = "重庆"
static main(args) {
def task = new HouseBookChromeCrawl();
task.run()
}
def run(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
def listPageUrl = getIndexPage(driver)
def keywords = getListPage(driver,"http://www.house-book.com.cn/"+listPageUrl)
keywords.remove("城南家园")
keywords.remove("康居西城")
println city +"共 "+keywords.size()+" 个小区"
getDistrictIndex(driver, keywords)
}
def getListPage(driver, url){
driver.get(url)
def cityDoc = Jsoup.parse(driver.getPageSource())
def total = JQueryUtils.selectText(".result-count-number:eq(0)", cityDoc)
if(total){
def page = getTotalPage(total, 6)
def keywords = getAllDist(driver, url, page);
return keywords
}
return null;
}
def getDistrictIndex(driver, keywords){
for(def word : keywords){
def code = URLEncoder.encode(URLEncoder.encode(word, "utf-8"),"utf-8")
goToDetailIndex(driver, code)
}
}
def getAllDist(driver, url, page){
def keywords = []
for(def i=1;i<=page;i++){
driver.get(url+"?selectPages="+i)
def doc = Jsoup.parse(driver.getPageSource())
def eles = JQueryUtils.select(".title-area .title-text", doc)
for(def e : eles){
def text = e.text()
if(text=="富州北郡"){
println i
}
keywords << text
}
}
return keywords
}
def getTotalPage(def total, def pageSize){
def mode = total.toInteger()/pageSize
if(mode > mode.toInteger()){
mode = mode.toInteger() +1
}
return mode;
}
def goToDetailIndex(driver, code){
def url = "http://www.house-book.com.cn/buildAction!pnameHome?pname="+code+"&price=&totalPrice=&bsqar=&htype=&btype=&sellKind="
driver.get(url)
WebDriverWait wait = new WebDriverWait(driver,20)
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("main"))));
def docment = Jsoup.parse(driver.getPageSource());
def dist_name = JQueryUtils.select(".info-pname", docment).text()
def addInfo = JQueryUtils.select(".addr-info", docment).text()
def region = null
if(addInfo.contains(":") && addInfo.contains("-")){
region = addInfo.substring(addInfo.indexOf(":")+1, addInfo.indexOf("-"))
}
def address = JQueryUtils.select(".addr-info span", docment).text()
// wait.until(new ExpectedCondition<WebElement>(){
// @Override
// public WebElement apply(WebDriver d){
// return d.findElement(By.linkText("[更多]"))
// }
// }).click()
driver.findElement(By.linkText("[更多]")).click()
// Thread.sleep(10)
// wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.id("shouUl"))));
def lis = checkElement(driver, "#shouUl .dongli");
def datas = []
if(lis){
for(def li : lis){
def id = li.attr("id")
def houseNum = li.select(".title-span").text()
def map = [:]
map.id = id
map.city = city
map.region = region
map.district_name = dist_name
map.address = address
map.house_num = houseNum
map.num = li.select(".dong_ul2 li:contains(许可证号:)").text().replace("许可证号:","")
map.type = li.select(".dong_ul2 .title-span-hbType").text()
def floor_heigth = li.select(".dong_ul2 label:contains(总楼层:)").text().replace("总楼层:","").replace("层","")
if(floor_heigth){
map.floor_heigth = floor_heigth
}
def room_count = li.select(".dong_ul2 label:contains(总户数:)").text()
if(room_count){
room_count = room_count.substring(room_count.indexOf("总户数:")+4, room_count.lastIndexOf("户"))
if(room_count){
map.household_count = room_count
}
}
datas << map
}
}
println dist_name +" 共 "+datas.size()+" 栋"
for(def m : datas){
try{
driver.findElement(By.id(m.id)).click()
def rows = parseDetail(driver)
println "栋divId: "+m.id +" 室:"+rows.size()
saveDate(m, rows)
}catch(Exception e){
println dist_name+" 栋divId: "+m.id+" 找不到相应的div element"
}
// driver.navigate().back();
}
}
def checkElement(driver, className){
def loop = 0
def stop = false
def lis = null
while (!stop){
loop++
def doc = Jsoup.parse(driver.getPageSource());
lis = JQueryUtils.select(className, doc)
if(lis || loop>=10){
stop = true;
}else{
Thread.sleep(5000)
}
}
return lis
}
def parseDetail(driver){
def trs = checkElement(driver, ".rooms tr");
def rows = []
for(def tr : trs){
def tds = tr.select("td")
if(tds){
def floor = null
for(def j=0;j<tds.size();j++){
if(j == 0){
floor = tds[0].text().replace("第","").replace("层","")
}else {
def td = tds[j]
def detail = [:]
def room_num = td.select("b").text()
if (room_num) {
detail.room_num = room_num
}
def area = td.select("span:contains(面积:)").text().replace("面积:", "").replace("㎡", "")
if (area) {
detail.area = area.toDouble()
}
def avg_price = td.select("span:contains(单价:)").text().replace("单价:", "").replace("元/㎡", "")
if (avg_price) {
detail.avg_price = avg_price.toInteger()
}
def house_type = td.select("span:contains(房屋类型:)").text().replace("房屋类型:", "")
if(house_type){
detail.estate_type = house_type
}
if(detail){
=-09ql.floor = floor
rows << detail
}
}
}
}
}
return rows;
}+*-*
def getIndexPage(driver){
driver.get("http://www.house-book.com.cn/mainAction/city/%25E9%2587%258D%25E5%25BA%2586%25E5%25B8%2582.html")
Thread.sleep(8000)
def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select("a:contains(新 房)", cityDoc)
if(cityEles){
def listPageUrl = cityEles[0].attr("href")
return listPageUrl
}
return null
}
def saveDate(m, rows){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", m.region)
q.put("district_name", m.district_name)
def datas = []
for(def map : rows){
q.put("house_num", m.house_num)
q.put("room_num", map.room_num)
q.put("floor_heigth" , m.floor_heigth)
q.put("floor", map.floor)
q.put("area", map.area)
def obj = loudongConn.findOne(q)
if(!obj){
def data = new BasicDBObject()
m.each{k,v->
data.put(k,v)
}
map.each{k,v->
data.put(k,v)
}
data.put("c_date", new Date())
datas << data
}
}
if(datas){
loudongConn.insert(datas)
}
}
}