149 lines
4.3 KiB
Java
149 lines
4.3 KiB
Java
|
|
import java.util.HashSet;
|
|
import java.util.Set;
|
|
import java.util.regex.Pattern;
|
|
import java.util.concurrent.TimeUnit;
|
|
|
|
import org.junit.*;
|
|
import org.jsoup.nodes.Element;
|
|
import org.jsoup.select.Elements;
|
|
|
|
import static org.junit.Assert.*;
|
|
import static org.hamcrest.CoreMatchers.*;
|
|
|
|
import org.openqa.selenium.*;
|
|
import org.openqa.selenium.firefox.FirefoxDriver;
|
|
import org.openqa.selenium.support.ui.ExpectedConditions;
|
|
import org.openqa.selenium.support.ui.Select;
|
|
import org.openqa.selenium.support.ui.WebDriverWait;
|
|
|
|
public class FetchPageToDB {
|
|
private WebDriver driver;
|
|
|
|
private boolean acceptNextAlert = true;
|
|
private StringBuffer verificationErrors = new StringBuffer();
|
|
private Set<String> URLset = new HashSet<String>();
|
|
|
|
private String baseUrl = "http://bol.szhome.com/";//基础url
|
|
private String firstEndUrl = "/search.aspx";//用基础url拼接成进入页面第一个URl
|
|
private String detailStartUrl = "";//把它作为前置Url拼接A标记的href,
|
|
static final String CITY = "深圳";//城市
|
|
static final String SITE_NAME = "深圳房地产信息网";//网站名称
|
|
static final String PAGE_NAME = "详细信息";//页面类型
|
|
static final String CATEGORY = "district";
|
|
static final String charset = "utf-8";//编码格式utf-8
|
|
static final String nextPageStr = ">";//下一页按钮的文本内容一般为:“下一页”or">"
|
|
static final int BreakPointpageCount = 1; //断点位置页数(即为开始页)
|
|
static final int pageCount = 200; //分页总数
|
|
static final String selectAeleStr = ".imgdiv a";//每页内容中选择到详细页面url标记
|
|
|
|
@Before
|
|
public void setUp() throws Exception {
|
|
|
|
driver = new FirefoxDriver();
|
|
driver.manage().timeouts().implicitlyWait(10, TimeUnit.SECONDS);
|
|
|
|
}
|
|
|
|
@Test
|
|
public void run() throws Exception {
|
|
|
|
driver.get(baseUrl + firstEndUrl);
|
|
|
|
dryRun(driver,BreakPointpageCount);//空跑到断点页面
|
|
fetchDetail(driver);//获取第一个页面的内容
|
|
|
|
System.out.printf("处理完成第%d页\r\n",BreakPointpageCount);
|
|
|
|
for(int i=BreakPointpageCount+1;i<=pageCount;i++) { //循环分页处理页面内容
|
|
|
|
Thread.sleep(500);
|
|
WebDriverWait webDriverWait = new WebDriverWait(driver, 10);
|
|
webDriverWait.until(ExpectedConditions.presenceOfElementLocated(By.linkText(nextPageStr))).click();
|
|
fetchDetail(driver);
|
|
System.out.printf("处理完成第%d页\r\n",i);
|
|
|
|
}
|
|
|
|
|
|
}
|
|
/**
|
|
* 空跑到上一个发生错误的页面
|
|
* @param driver
|
|
* @param BreakPointpageCount
|
|
* @throws InterruptedException
|
|
*/
|
|
public void dryRun(WebDriver driver,int BreakPointpageCount) throws InterruptedException{
|
|
if(BreakPointpageCount-1 == 0){
|
|
return ;
|
|
}
|
|
for(int i=0;i<BreakPointpageCount-1;i++){
|
|
|
|
Thread.sleep(500);
|
|
WebDriverWait webDriverWait = new WebDriverWait(driver, 10);
|
|
webDriverWait.until(ExpectedConditions.presenceOfElementLocated(By.linkText(nextPageStr))).click();
|
|
Thread.sleep(500);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* 获取页面详细内容保存到数据库
|
|
* @param driver
|
|
*/
|
|
public void fetchDetail(WebDriver driver){
|
|
// System.out.println( driver.getPageSource());
|
|
Elements leles = JQueryUtils.select(selectAeleStr, driver.getPageSource());
|
|
for (Element link:leles){
|
|
|
|
String detailUrl = detailStartUrl + link.attr("href");
|
|
if(URLset.contains(detailUrl)){
|
|
continue;
|
|
}
|
|
String detailContent = InvokeHttpRequest.get(detailUrl, charset,null);
|
|
URLset.add(detailUrl);
|
|
SaveToWebpage.DataSaveToWebpage(CITY, SITE_NAME, PAGE_NAME, CATEGORY, detailUrl, detailContent);//保存数据
|
|
}
|
|
}
|
|
@After
|
|
public void tearDown() throws Exception {
|
|
driver.quit();
|
|
String verificationErrorString = verificationErrors.toString();
|
|
if (!"".equals(verificationErrorString)) {
|
|
fail(verificationErrorString);
|
|
}
|
|
}
|
|
|
|
private boolean isElementPresent(By by) {
|
|
try {
|
|
driver.findElement(by);
|
|
return true;
|
|
} catch (NoSuchElementException e) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
private boolean isAlertPresent() {
|
|
try {
|
|
driver.switchTo().alert();
|
|
return true;
|
|
} catch (NoAlertPresentException e) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
private String closeAlertAndGetItsText() {
|
|
try {
|
|
Alert alert = driver.switchTo().alert();
|
|
String alertText = alert.getText();
|
|
if (acceptNextAlert) {
|
|
alert.accept();
|
|
} else {
|
|
alert.dismiss();
|
|
}
|
|
return alertText;
|
|
} finally {
|
|
acceptNextAlert = true;
|
|
}
|
|
}
|
|
}
|