Files
main/projects/Selenium/src/test/java/FetchPageToDB.java
T
2021-03-13 13:48:21 +08:00

149 lines
4.3 KiB
Java

import java.util.HashSet;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.concurrent.TimeUnit;
import org.junit.*;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import static org.junit.Assert.*;
import static org.hamcrest.CoreMatchers.*;
import org.openqa.selenium.*;
import org.openqa.selenium.firefox.FirefoxDriver;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.Select;
import org.openqa.selenium.support.ui.WebDriverWait;
public class FetchPageToDB {
private WebDriver driver;
private boolean acceptNextAlert = true;
private StringBuffer verificationErrors = new StringBuffer();
private Set<String> URLset = new HashSet<String>();
private String baseUrl = "http://bol.szhome.com/";//基础url
private String firstEndUrl = "/search.aspx";//用基础url拼接成进入页面第一个URl
private String detailStartUrl = "";//把它作为前置Url拼接A标记的href,
static final String CITY = "深圳";//城市
static final String SITE_NAME = "深圳房地产信息网";//网站名称
static final String PAGE_NAME = "详细信息";//页面类型
static final String CATEGORY = "district";
static final String charset = "utf-8";//编码格式utf-8
static final String nextPageStr = ">";//下一页按钮的文本内容一般为:“下一页”or">"
static final int BreakPointpageCount = 1; //断点位置页数(即为开始页)
static final int pageCount = 200; //分页总数
static final String selectAeleStr = ".imgdiv a";//每页内容中选择到详细页面url标记
@Before
public void setUp() throws Exception {
driver = new FirefoxDriver();
driver.manage().timeouts().implicitlyWait(10, TimeUnit.SECONDS);
}
@Test
public void run() throws Exception {
driver.get(baseUrl + firstEndUrl);
dryRun(driver,BreakPointpageCount);//空跑到断点页面
fetchDetail(driver);//获取第一个页面的内容
System.out.printf("处理完成第%d页\r\n",BreakPointpageCount);
for(int i=BreakPointpageCount+1;i<=pageCount;i++) { //循环分页处理页面内容
Thread.sleep(500);
WebDriverWait webDriverWait = new WebDriverWait(driver, 10);
webDriverWait.until(ExpectedConditions.presenceOfElementLocated(By.linkText(nextPageStr))).click();
fetchDetail(driver);
System.out.printf("处理完成第%d页\r\n",i);
}
}
/**
* 空跑到上一个发生错误的页面
* @param driver
* @param BreakPointpageCount
* @throws InterruptedException
*/
public void dryRun(WebDriver driver,int BreakPointpageCount) throws InterruptedException{
if(BreakPointpageCount-1 == 0){
return ;
}
for(int i=0;i<BreakPointpageCount-1;i++){
Thread.sleep(500);
WebDriverWait webDriverWait = new WebDriverWait(driver, 10);
webDriverWait.until(ExpectedConditions.presenceOfElementLocated(By.linkText(nextPageStr))).click();
Thread.sleep(500);
}
}
/**
* 获取页面详细内容保存到数据库
* @param driver
*/
public void fetchDetail(WebDriver driver){
// System.out.println( driver.getPageSource());
Elements leles = JQueryUtils.select(selectAeleStr, driver.getPageSource());
for (Element link:leles){
String detailUrl = detailStartUrl + link.attr("href");
if(URLset.contains(detailUrl)){
continue;
}
String detailContent = InvokeHttpRequest.get(detailUrl, charset,null);
URLset.add(detailUrl);
SaveToWebpage.DataSaveToWebpage(CITY, SITE_NAME, PAGE_NAME, CATEGORY, detailUrl, detailContent);//保存数据
}
}
@After
public void tearDown() throws Exception {
driver.quit();
String verificationErrorString = verificationErrors.toString();
if (!"".equals(verificationErrorString)) {
fail(verificationErrorString);
}
}
private boolean isElementPresent(By by) {
try {
driver.findElement(by);
return true;
} catch (NoSuchElementException e) {
return false;
}
}
private boolean isAlertPresent() {
try {
driver.switchTo().alert();
return true;
} catch (NoAlertPresentException e) {
return false;
}
}
private String closeAlertAndGetItsText() {
try {
Alert alert = driver.switchTo().alert();
String alertText = alert.getText();
if (acceptNextAlert) {
alert.accept();
} else {
alert.dismiss();
}
return alertText;
} finally {
acceptNextAlert = true;
}
}
}