修改代码字符编码

This commit is contained in:
2026-02-24 17:53:50 +08:00
parent 2495fe5f6d
commit c8418c7d4b
5195 changed files with 121754 additions and 121754 deletions
@@ -24,18 +24,18 @@ public class FetchPageToDB {
private StringBuffer verificationErrors = new StringBuffer();
private Set<String> URLset = new HashSet<String>();
private String baseUrl = "http://bol.szhome.com/";//基础url
private String firstEndUrl = "/search.aspx";//用基础url拼接成进入页面第一个URl
private String detailStartUrl = "";//把它作为前置Url拼接A标记的href,
static final String CITY = "深圳";//城市
static final String SITE_NAME = "深圳房地产信息网";//网站名称
static final String PAGE_NAME = "详细信息";//页面类型
private String baseUrl = "http://bol.szhome.com/";//基础url
private String firstEndUrl = "/search.aspx";//用基础url拼接成进入页面第一个URl
private String detailStartUrl = "";//把它作为前置Url拼接A标记的href,
static final String CITY = "深圳";//城市
static final String SITE_NAME = "深圳房地产信息网";//网站名称
static final String PAGE_NAME = "详细信息";//页面类型
static final String CATEGORY = "district";
static final String charset = "utf-8";//编码格式utf-8
static final String nextPageStr = ">";//下一页按钮的文本内容一般为:“下一页”or">"
static final int BreakPointpageCount = 1; //断点位置页数(即为开始页)
static final int pageCount = 200; //分页总数
static final String selectAeleStr = ".imgdiv a";//每页内容中选择到详细页面url标记
static final String charset = "utf-8";//编码格式utf-8
static final String nextPageStr = ">";//下一页按钮的文本内容一般为:“下一页”or">"
static final int BreakPointpageCount = 1; //断点位置页数(即为开始页)
static final int pageCount = 200; //分页总数
static final String selectAeleStr = ".imgdiv a";//每页内容中选择到详细页面url标记
@Before
public void setUp() throws Exception {
@@ -50,25 +50,25 @@ public class FetchPageToDB {
driver.get(baseUrl + firstEndUrl);
dryRun(driver,BreakPointpageCount);//空跑到断点页面
fetchDetail(driver);//获取第一个页面的内容
dryRun(driver,BreakPointpageCount);//空跑到断点页面
fetchDetail(driver);//获取第一个页面的内容
System.out.printf("处理完成第%d页\r\n",BreakPointpageCount);
System.out.printf("处理完成第%d页\r\n",BreakPointpageCount);
for(int i=BreakPointpageCount+1;i<=pageCount;i++) { //循环分页处理页面内容
for(int i=BreakPointpageCount+1;i<=pageCount;i++) { //循环分页处理页面内容
Thread.sleep(500);
WebDriverWait webDriverWait = new WebDriverWait(driver, 10);
webDriverWait.until(ExpectedConditions.presenceOfElementLocated(By.linkText(nextPageStr))).click();
fetchDetail(driver);
System.out.printf("处理完成第%d页\r\n",i);
System.out.printf("处理完成第%d页\r\n",i);
}
}
/**
* 空跑到上一个发生错误的页面
* 空跑到上一个发生错误的页面
* @param driver
* @param BreakPointpageCount
* @throws InterruptedException
@@ -87,7 +87,7 @@ public class FetchPageToDB {
}
/**
* 获取页面详细内容保存到数据库
* 获取页面详细内容保存到数据库
* @param driver
*/
public void fetchDetail(WebDriver driver){
@@ -101,7 +101,7 @@ public class FetchPageToDB {
}
String detailContent = InvokeHttpRequest.get(detailUrl, charset,null);
URLset.add(detailUrl);
SaveToWebpage.DataSaveToWebpage(CITY, SITE_NAME, PAGE_NAME, CATEGORY, detailUrl, detailContent);//保存数据
SaveToWebpage.DataSaveToWebpage(CITY, SITE_NAME, PAGE_NAME, CATEGORY, detailUrl, detailContent);//保存数据
}
}
@After
@@ -22,7 +22,7 @@ public class FetchPageUtil {
/**
* 打印传入页面渲染后的源码
* 打印传入页面渲染后的源码
* @param url
* @throws InterruptedException
*/
@@ -33,7 +33,7 @@ public class FetchPageUtil {
driver.quit();
}
/**
* 打印页面源码和选择器匹配后的结果
* 打印页面源码和选择器匹配后的结果
* @param url
* @param selectStr
* @throws InterruptedException
@@ -46,7 +46,7 @@ public class FetchPageUtil {
driver.quit();
}
/**
* 测试分页是否正常运行
* 测试分页是否正常运行
* @param url
* @param nextStr
* @throws InterruptedException
@@ -75,8 +75,8 @@ public class FetchPageUtil {
}
@Test
public void runMain() throws InterruptedException{
//printPageSource("http://bol.szhome.com/search.aspx");//第一步,查看分页的渲染后源码
printselects("http://bol.szhome.com/search.aspx",".imgdiv a");//根据源码写出选择器String
//TestNext("http://bol.szhome.com/search.aspx",">");//测试分页
//printPageSource("http://bol.szhome.com/search.aspx");//第一步,查看分页的渲染后源码
printselects("http://bol.szhome.com/search.aspx",".imgdiv a");//根据源码写出选择器String
//TestNext("http://bol.szhome.com/search.aspx",">");//测试分页
}
}
@@ -46,7 +46,7 @@ public class JQueryUtils {
try{
if(repeatTimes>0){
Thread.sleep(1000);
System.out.println( "准备尝试第"+(repeatTimes+1)+"次请求:$url");
System.out.println( "准备尝试第"+(repeatTimes+1)+"次请求:$url");
}
doc = Jsoup.parse(new URL(url), 5*1000);
if(doc != null){
@@ -58,7 +58,7 @@ public class JQueryUtils {
}
if(doc==null){
System.out.println("检测到不可访问链接:$url");
System.out.println("检测到不可访问链接:$url");
}
return doc;
}
@@ -69,7 +69,7 @@ public class JQueryUtils {
try{
if(repeatTimes>0){
Thread.sleep(1000);
System.out.println("准备尝试第"+(repeatTimes+1)+"次请求:$url");
System.out.println("准备尝试第"+(repeatTimes+1)+"次请求:$url");
}
URL _url = new URL(url);
input = _url.openStream();
@@ -92,7 +92,7 @@ public class JQueryUtils {
}
if(doc==null){
System.out.println("检测到不可访问链接:$url");
System.out.println("检测到不可访问链接:$url");
}
return doc;
}
@@ -70,7 +70,7 @@ public class SaveToWebpage {
}
/**
* ѹËõ
* 压缩
*
* @param content
* @return