新增贝壳找房城市种子

This commit is contained in:
沈健
2021-06-02 13:44:09 +08:00
parent 62a8443160
commit ea3ae0cf61
3 changed files with 78 additions and 4 deletions
@@ -1,6 +1,7 @@
package com.fangjia.script.temp.shenjian.crawl
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.temp.shenjian.util.CityUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
@@ -252,16 +253,13 @@ class City58 {
" }";
static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018)
static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017)
static Set citySet = CityUtils.listCitySet()
static void main(String[] args) {
JSONObject jsonObject = JSONObject.parseObject(cityList)
Set<String> provinceSet = jsonObject.keySet()
List<DBObject> ret = new ArrayList<>()
List<String> list = cityMapConn.distinct("fangjia_city")
Set<String> citySet = new HashSet<>(list);
for (String province: provinceSet) {
JSONObject provinceInfo = jsonObject[province]
for (String city : provinceInfo.keySet()) {
@@ -0,0 +1,57 @@
package com.fangjia.script.temp.shenjian.crawl.ke
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.temp.shenjian.util.CityUtils
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.lang3.StringUtils
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import org.jsoup.select.Elements
/**
* 贝壳找房城市种子生成
*
* @author shenjian
* @since 2021/6/2
*/
class CityKe {
static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018)
static Set citySet = CityUtils.listCitySet()
static void main(String[] args) {
final def URL = "https://www.ke.com/city/"
String res = ABuYunUtil.getReponseContent2(URL, "utf-8")
if (res) {
Document doc = Jsoup.parse(res);
Elements elements = JQueryUtils.select(".city_list_ul .city_list_li", doc)
List<DBObject> ret = new ArrayList<>()
for (Element element : elements) {
String province = JQueryUtils.selectText(".city_province .city_list_tit", element)
Elements cityElements = JQueryUtils.select(".CLICKDATA a", element)
for (Element cityElement: cityElements) {
String city = cityElement.text()
if (citySet.contains(city)) {
String url = cityElement.attr("href")
String cityShort = StringUtils.substringBetween(url,"//", ".ke.com")
DBObject dbObject = new BasicDBObject()
dbObject.put("province", province)
dbObject.put("city", city)
dbObject.put("city_short", cityShort)
dbObject.put("c_date", new Date())
dbObject.put("m_date", new Date())
dbObject.put("root", "ke.com")
ret.add(dbObject)
}
}
}
cityConn.insert(ret)
println("贝壳找房城市数据写入成功")
}
}
}
@@ -0,0 +1,19 @@
package com.fangjia.script.temp.shenjian.util
import com.fangjia.script.util.MongoUtils
import com.mongodb.DBCollection
/**
* @author shenjian
* @since 2021/6/2
*/
class CityUtils {
static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017)
static listCitySet() {
List<String> list = cityMapConn.distinct("fangjia_city")
Set<String> citySet = new HashSet<>(list);
return citySet
}
}