import java.util.HashSet; import java.util.Set; import java.util.regex.Pattern; import java.util.concurrent.TimeUnit; import org.junit.*; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import static org.junit.Assert.*; import static org.hamcrest.CoreMatchers.*; import org.openqa.selenium.*; import org.openqa.selenium.firefox.FirefoxDriver; import org.openqa.selenium.support.ui.ExpectedConditions; import org.openqa.selenium.support.ui.Select; import org.openqa.selenium.support.ui.WebDriverWait; public class FetchPageToDB { private WebDriver driver; private boolean acceptNextAlert = true; private StringBuffer verificationErrors = new StringBuffer(); private Set URLset = new HashSet(); private String baseUrl = "http://bol.szhome.com/";//基础url private String firstEndUrl = "/search.aspx";//用基础url拼接成进入页面第一个URl private String detailStartUrl = "";//把它作为前置Url拼接A标记的href, static final String CITY = "深圳";//城市 static final String SITE_NAME = "深圳房地产信息网";//网站名称 static final String PAGE_NAME = "详细信息";//页面类型 static final String CATEGORY = "district"; static final String charset = "utf-8";//编码格式utf-8 static final String nextPageStr = ">";//下一页按钮的文本内容一般为:“下一页”or">" static final int BreakPointpageCount = 1; //断点位置页数(即为开始页) static final int pageCount = 200; //分页总数 static final String selectAeleStr = ".imgdiv a";//每页内容中选择到详细页面url标记 @Before public void setUp() throws Exception { driver = new FirefoxDriver(); driver.manage().timeouts().implicitlyWait(10, TimeUnit.SECONDS); } @Test public void run() throws Exception { driver.get(baseUrl + firstEndUrl); dryRun(driver,BreakPointpageCount);//空跑到断点页面 fetchDetail(driver);//获取第一个页面的内容 System.out.printf("处理完成第%d页\r\n",BreakPointpageCount); for(int i=BreakPointpageCount+1;i<=pageCount;i++) { //循环分页处理页面内容 Thread.sleep(500); WebDriverWait webDriverWait = new WebDriverWait(driver, 10); webDriverWait.until(ExpectedConditions.presenceOfElementLocated(By.linkText(nextPageStr))).click(); fetchDetail(driver); System.out.printf("处理完成第%d页\r\n",i); } } /** * 空跑到上一个发生错误的页面 * @param driver * @param BreakPointpageCount * @throws InterruptedException */ public void dryRun(WebDriver driver,int BreakPointpageCount) throws InterruptedException{ if(BreakPointpageCount-1 == 0){ return ; } for(int i=0;i