新增房天下城市种子
This commit is contained in:
+56
@@ -0,0 +1,56 @@
|
||||
package com.fangjia.script.temp.shenjian.crawl.ke
|
||||
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.temp.shenjian.util.CityUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.commons.lang3.StringUtils
|
||||
import org.jsoup.Jsoup
|
||||
import org.jsoup.nodes.Document
|
||||
import org.jsoup.nodes.Element
|
||||
import org.jsoup.select.Elements
|
||||
|
||||
/**
|
||||
* 贝壳找房城市种子生成
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2021/6/2
|
||||
*/
|
||||
class CityFang {
|
||||
|
||||
static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018)
|
||||
static Set citySet = CityUtils.listCitySet()
|
||||
|
||||
static void main(String[] args) {
|
||||
// 房天下较为特殊,直接从url取乱码,需要改页面所有非UTF-8编码为UTF-8
|
||||
Document doc = Jsoup.parse(new File("F:\\数据\\爬虫\\fang_city.html"), "UTF-8");
|
||||
Elements elements = JQueryUtils.select(".outCont li", doc)
|
||||
List<DBObject> ret = new ArrayList<>()
|
||||
for (Element element : elements) {
|
||||
String province = JQueryUtils.selectText("strong", element)
|
||||
Elements cityElements = JQueryUtils.select("a", element)
|
||||
for (Element cityElement: cityElements) {
|
||||
String city = cityElement.text()
|
||||
if ("直辖市".equals(province)) {
|
||||
province = city
|
||||
}
|
||||
if (citySet.contains(city)) {
|
||||
String url = cityElement.attr("href")
|
||||
String cityShort = StringUtils.substringBetween(url,"//", ".esf.fang.com")
|
||||
DBObject dbObject = new BasicDBObject()
|
||||
dbObject.put("province", province)
|
||||
dbObject.put("city", city)
|
||||
dbObject.put("city_short", cityShort)
|
||||
dbObject.put("c_date", new Date())
|
||||
dbObject.put("m_date", new Date())
|
||||
dbObject.put("root", "fang.com")
|
||||
ret.add(dbObject)
|
||||
}
|
||||
}
|
||||
}
|
||||
cityConn.insert(ret)
|
||||
println("房天下城市数据写入成功")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user