diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/fang/CityFang.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/fang/CityFang.groovy new file mode 100644 index 000000000..88a3b6f38 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/fang/CityFang.groovy @@ -0,0 +1,56 @@ +package com.fangjia.script.temp.shenjian.crawl.ke + +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.temp.shenjian.util.CityUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.lang3.StringUtils +import org.jsoup.Jsoup +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import org.jsoup.select.Elements + +/** + * 贝壳找房城市种子生成 + * + * @author shenjian + * @since 2021/6/2 + */ +class CityFang { + + static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018) + static Set citySet = CityUtils.listCitySet() + + static void main(String[] args) { + // 房天下较为特殊,直接从url取乱码,需要改页面所有非UTF-8编码为UTF-8 + Document doc = Jsoup.parse(new File("F:\\数据\\爬虫\\fang_city.html"), "UTF-8"); + Elements elements = JQueryUtils.select(".outCont li", doc) + List ret = new ArrayList<>() + for (Element element : elements) { + String province = JQueryUtils.selectText("strong", element) + Elements cityElements = JQueryUtils.select("a", element) + for (Element cityElement: cityElements) { + String city = cityElement.text() + if ("直辖市".equals(province)) { + province = city + } + if (citySet.contains(city)) { + String url = cityElement.attr("href") + String cityShort = StringUtils.substringBetween(url,"//", ".esf.fang.com") + DBObject dbObject = new BasicDBObject() + dbObject.put("province", province) + dbObject.put("city", city) + dbObject.put("city_short", cityShort) + dbObject.put("c_date", new Date()) + dbObject.put("m_date", new Date()) + dbObject.put("root", "fang.com") + ret.add(dbObject) + } + } + } + cityConn.insert(ret) + println("房天下城市数据写入成功") + } +}