diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/City58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/City58.groovy index 8b9abf395..5222957f7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/City58.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/City58.groovy @@ -1,6 +1,7 @@ package com.fangjia.script.temp.shenjian.crawl import com.alibaba.fastjson.JSONObject +import com.fangjia.script.temp.shenjian.util.CityUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection @@ -252,16 +253,13 @@ class City58 { " }"; static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018) - static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) + static Set citySet = CityUtils.listCitySet() static void main(String[] args) { JSONObject jsonObject = JSONObject.parseObject(cityList) Set provinceSet = jsonObject.keySet() List ret = new ArrayList<>() - List list = cityMapConn.distinct("fangjia_city") - Set citySet = new HashSet<>(list); - for (String province: provinceSet) { JSONObject provinceInfo = jsonObject[province] for (String city : provinceInfo.keySet()) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CityKe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CityKe.groovy new file mode 100644 index 000000000..e8ae7fc0c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CityKe.groovy @@ -0,0 +1,57 @@ +package com.fangjia.script.temp.shenjian.crawl.ke + +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.temp.shenjian.util.CityUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.lang3.StringUtils +import org.jsoup.Jsoup +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import org.jsoup.select.Elements + +/** + * 贝壳找房城市种子生成 + * + * @author shenjian + * @since 2021/6/2 + */ +class CityKe { + + static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018) + static Set citySet = CityUtils.listCitySet() + + static void main(String[] args) { + final def URL = "https://www.ke.com/city/" + String res = ABuYunUtil.getReponseContent2(URL, "utf-8") + if (res) { + Document doc = Jsoup.parse(res); + Elements elements = JQueryUtils.select(".city_list_ul .city_list_li", doc) + List ret = new ArrayList<>() + for (Element element : elements) { + String province = JQueryUtils.selectText(".city_province .city_list_tit", element) + Elements cityElements = JQueryUtils.select(".CLICKDATA a", element) + for (Element cityElement: cityElements) { + String city = cityElement.text() + if (citySet.contains(city)) { + String url = cityElement.attr("href") + String cityShort = StringUtils.substringBetween(url,"//", ".ke.com") + DBObject dbObject = new BasicDBObject() + dbObject.put("province", province) + dbObject.put("city", city) + dbObject.put("city_short", cityShort) + dbObject.put("c_date", new Date()) + dbObject.put("m_date", new Date()) + dbObject.put("root", "ke.com") + ret.add(dbObject) + } + } + } + cityConn.insert(ret) + println("贝壳找房城市数据写入成功") + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/CityUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/CityUtils.groovy new file mode 100644 index 000000000..2949d3803 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/CityUtils.groovy @@ -0,0 +1,19 @@ +package com.fangjia.script.temp.shenjian.util + +import com.fangjia.script.util.MongoUtils +import com.mongodb.DBCollection + +/** + * @author shenjian + * @since 2021/6/2 + */ +class CityUtils { + + static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) + + static listCitySet() { + List list = cityMapConn.distinct("fangjia_city") + Set citySet = new HashSet<>(list); + return citySet + } +}