修改代码字符编码
This commit is contained in:
@@ -24,18 +24,18 @@ public class FetchPageToDB {
|
||||
private StringBuffer verificationErrors = new StringBuffer();
|
||||
private Set<String> URLset = new HashSet<String>();
|
||||
|
||||
private String baseUrl = "http://bol.szhome.com/";//基础url
|
||||
private String firstEndUrl = "/search.aspx";//用基础url拼接成进入页面第一个URl
|
||||
private String detailStartUrl = "";//把它作为前置Url拼接A标记的href,
|
||||
static final String CITY = "深圳";//城市
|
||||
static final String SITE_NAME = "深圳房地产信息网";//网站名称
|
||||
static final String PAGE_NAME = "详细信息";//页面类型
|
||||
private String baseUrl = "http://bol.szhome.com/";//基础url
|
||||
private String firstEndUrl = "/search.aspx";//用基础url拼接成进入页面第一个URl
|
||||
private String detailStartUrl = "";//把它作为前置Url拼接A标记的href,
|
||||
static final String CITY = "深圳";//城市
|
||||
static final String SITE_NAME = "深圳房地产信息网";//网站名称
|
||||
static final String PAGE_NAME = "详细信息";//页面类型
|
||||
static final String CATEGORY = "district";
|
||||
static final String charset = "utf-8";//编码格式utf-8
|
||||
static final String nextPageStr = ">";//下一页按钮的文本内容一般为:“下一页”or">"
|
||||
static final int BreakPointpageCount = 1; //断点位置页数(即为开始页)
|
||||
static final int pageCount = 200; //分页总数
|
||||
static final String selectAeleStr = ".imgdiv a";//每页内容中选择到详细页面url标记
|
||||
static final String charset = "utf-8";//编码格式utf-8
|
||||
static final String nextPageStr = ">";//下一页按钮的文本内容一般为:“下一页”or">"
|
||||
static final int BreakPointpageCount = 1; //断点位置页数(即为开始页)
|
||||
static final int pageCount = 200; //分页总数
|
||||
static final String selectAeleStr = ".imgdiv a";//每页内容中选择到详细页面url标记
|
||||
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
@@ -50,25 +50,25 @@ public class FetchPageToDB {
|
||||
|
||||
driver.get(baseUrl + firstEndUrl);
|
||||
|
||||
dryRun(driver,BreakPointpageCount);//空跑到断点页面
|
||||
fetchDetail(driver);//获取第一个页面的内容
|
||||
dryRun(driver,BreakPointpageCount);//空跑到断点页面
|
||||
fetchDetail(driver);//获取第一个页面的内容
|
||||
|
||||
System.out.printf("处理完成第%d页\r\n",BreakPointpageCount);
|
||||
System.out.printf("处理完成第%d页\r\n",BreakPointpageCount);
|
||||
|
||||
for(int i=BreakPointpageCount+1;i<=pageCount;i++) { //循环分页处理页面内容
|
||||
for(int i=BreakPointpageCount+1;i<=pageCount;i++) { //循环分页处理页面内容
|
||||
|
||||
Thread.sleep(500);
|
||||
WebDriverWait webDriverWait = new WebDriverWait(driver, 10);
|
||||
webDriverWait.until(ExpectedConditions.presenceOfElementLocated(By.linkText(nextPageStr))).click();
|
||||
fetchDetail(driver);
|
||||
System.out.printf("处理完成第%d页\r\n",i);
|
||||
System.out.printf("处理完成第%d页\r\n",i);
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
/**
|
||||
* 空跑到上一个发生错误的页面
|
||||
* 空跑到上一个发生错误的页面
|
||||
* @param driver
|
||||
* @param BreakPointpageCount
|
||||
* @throws InterruptedException
|
||||
@@ -87,7 +87,7 @@ public class FetchPageToDB {
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取页面详细内容保存到数据库
|
||||
* 获取页面详细内容保存到数据库
|
||||
* @param driver
|
||||
*/
|
||||
public void fetchDetail(WebDriver driver){
|
||||
@@ -101,7 +101,7 @@ public class FetchPageToDB {
|
||||
}
|
||||
String detailContent = InvokeHttpRequest.get(detailUrl, charset,null);
|
||||
URLset.add(detailUrl);
|
||||
SaveToWebpage.DataSaveToWebpage(CITY, SITE_NAME, PAGE_NAME, CATEGORY, detailUrl, detailContent);//保存数据
|
||||
SaveToWebpage.DataSaveToWebpage(CITY, SITE_NAME, PAGE_NAME, CATEGORY, detailUrl, detailContent);//保存数据
|
||||
}
|
||||
}
|
||||
@After
|
||||
|
||||
@@ -22,7 +22,7 @@ public class FetchPageUtil {
|
||||
|
||||
|
||||
/**
|
||||
* 打印传入页面渲染后的源码
|
||||
* 打印传入页面渲染后的源码
|
||||
* @param url
|
||||
* @throws InterruptedException
|
||||
*/
|
||||
@@ -33,7 +33,7 @@ public class FetchPageUtil {
|
||||
driver.quit();
|
||||
}
|
||||
/**
|
||||
* 打印页面源码和选择器匹配后的结果
|
||||
* 打印页面源码和选择器匹配后的结果
|
||||
* @param url
|
||||
* @param selectStr
|
||||
* @throws InterruptedException
|
||||
@@ -46,7 +46,7 @@ public class FetchPageUtil {
|
||||
driver.quit();
|
||||
}
|
||||
/**
|
||||
* 测试分页是否正常运行
|
||||
* 测试分页是否正常运行
|
||||
* @param url
|
||||
* @param nextStr
|
||||
* @throws InterruptedException
|
||||
@@ -75,8 +75,8 @@ public class FetchPageUtil {
|
||||
}
|
||||
@Test
|
||||
public void runMain() throws InterruptedException{
|
||||
//printPageSource("http://bol.szhome.com/search.aspx");//第一步,查看分页的渲染后源码
|
||||
printselects("http://bol.szhome.com/search.aspx",".imgdiv a");//根据源码写出选择器String
|
||||
//TestNext("http://bol.szhome.com/search.aspx",">");//测试分页
|
||||
//printPageSource("http://bol.szhome.com/search.aspx");//第一步,查看分页的渲染后源码
|
||||
printselects("http://bol.szhome.com/search.aspx",".imgdiv a");//根据源码写出选择器String
|
||||
//TestNext("http://bol.szhome.com/search.aspx",">");//测试分页
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,7 +46,7 @@ public class JQueryUtils {
|
||||
try{
|
||||
if(repeatTimes>0){
|
||||
Thread.sleep(1000);
|
||||
System.out.println( "准备尝试第"+(repeatTimes+1)+"次请求:$url");
|
||||
System.out.println( "准备尝试第"+(repeatTimes+1)+"次请求:$url");
|
||||
}
|
||||
doc = Jsoup.parse(new URL(url), 5*1000);
|
||||
if(doc != null){
|
||||
@@ -58,7 +58,7 @@ public class JQueryUtils {
|
||||
|
||||
}
|
||||
if(doc==null){
|
||||
System.out.println("检测到不可访问链接:$url");
|
||||
System.out.println("检测到不可访问链接:$url");
|
||||
}
|
||||
return doc;
|
||||
}
|
||||
@@ -69,7 +69,7 @@ public class JQueryUtils {
|
||||
try{
|
||||
if(repeatTimes>0){
|
||||
Thread.sleep(1000);
|
||||
System.out.println("准备尝试第"+(repeatTimes+1)+"次请求:$url");
|
||||
System.out.println("准备尝试第"+(repeatTimes+1)+"次请求:$url");
|
||||
}
|
||||
URL _url = new URL(url);
|
||||
input = _url.openStream();
|
||||
@@ -92,7 +92,7 @@ public class JQueryUtils {
|
||||
|
||||
}
|
||||
if(doc==null){
|
||||
System.out.println("检测到不可访问链接:$url");
|
||||
System.out.println("检测到不可访问链接:$url");
|
||||
}
|
||||
return doc;
|
||||
}
|
||||
|
||||
@@ -70,7 +70,7 @@ public class SaveToWebpage {
|
||||
|
||||
}
|
||||
/**
|
||||
* ѹËõ
|
||||
* 压缩
|
||||
*
|
||||
* @param content
|
||||
* @return
|
||||
|
||||
Reference in New Issue
Block a user