新增房天下城市种子

This commit is contained in:
沈健
2021-06-02 16:53:12 +08:00
parent 62a419ea66
commit 37d0ad878c
@@ -0,0 +1,56 @@
package com.fangjia.script.temp.shenjian.crawl.ke
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.temp.shenjian.util.CityUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.lang3.StringUtils
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import org.jsoup.select.Elements
/**
* 贝壳找房城市种子生成
*
* @author shenjian
* @since 2021/6/2
*/
class CityFang {
static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018)
static Set citySet = CityUtils.listCitySet()
static void main(String[] args) {
// 房天下较为特殊,直接从url取乱码,需要改页面所有非UTF-8编码为UTF-8
Document doc = Jsoup.parse(new File("F:\\数据\\爬虫\\fang_city.html"), "UTF-8");
Elements elements = JQueryUtils.select(".outCont li", doc)
List<DBObject> ret = new ArrayList<>()
for (Element element : elements) {
String province = JQueryUtils.selectText("strong", element)
Elements cityElements = JQueryUtils.select("a", element)
for (Element cityElement: cityElements) {
String city = cityElement.text()
if ("直辖市".equals(province)) {
province = city
}
if (citySet.contains(city)) {
String url = cityElement.attr("href")
String cityShort = StringUtils.substringBetween(url,"//", ".esf.fang.com")
DBObject dbObject = new BasicDBObject()
dbObject.put("province", province)
dbObject.put("city", city)
dbObject.put("city_short", cityShort)
dbObject.put("c_date", new Date())
dbObject.put("m_date", new Date())
dbObject.put("root", "fang.com")
ret.add(dbObject)
}
}
}
cityConn.insert(ret)
println("房天下城市数据写入成功")
}
}