新增贝壳找房城市种子
This commit is contained in:
+2
-4
@@ -1,6 +1,7 @@
|
||||
package com.fangjia.script.temp.shenjian.crawl
|
||||
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.script.temp.shenjian.util.CityUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
@@ -252,16 +253,13 @@ class City58 {
|
||||
" }";
|
||||
|
||||
static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018)
|
||||
static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017)
|
||||
static Set citySet = CityUtils.listCitySet()
|
||||
|
||||
static void main(String[] args) {
|
||||
JSONObject jsonObject = JSONObject.parseObject(cityList)
|
||||
Set<String> provinceSet = jsonObject.keySet()
|
||||
List<DBObject> ret = new ArrayList<>()
|
||||
|
||||
List<String> list = cityMapConn.distinct("fangjia_city")
|
||||
Set<String> citySet = new HashSet<>(list);
|
||||
|
||||
for (String province: provinceSet) {
|
||||
JSONObject provinceInfo = jsonObject[province]
|
||||
for (String city : provinceInfo.keySet()) {
|
||||
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
package com.fangjia.script.temp.shenjian.crawl.ke
|
||||
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.temp.shenjian.util.CityUtils
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.commons.lang3.StringUtils
|
||||
import org.jsoup.Jsoup
|
||||
import org.jsoup.nodes.Document
|
||||
import org.jsoup.nodes.Element
|
||||
import org.jsoup.select.Elements
|
||||
|
||||
/**
|
||||
* 贝壳找房城市种子生成
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2021/6/2
|
||||
*/
|
||||
class CityKe {
|
||||
|
||||
static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018)
|
||||
static Set citySet = CityUtils.listCitySet()
|
||||
|
||||
static void main(String[] args) {
|
||||
final def URL = "https://www.ke.com/city/"
|
||||
String res = ABuYunUtil.getReponseContent2(URL, "utf-8")
|
||||
if (res) {
|
||||
Document doc = Jsoup.parse(res);
|
||||
Elements elements = JQueryUtils.select(".city_list_ul .city_list_li", doc)
|
||||
List<DBObject> ret = new ArrayList<>()
|
||||
for (Element element : elements) {
|
||||
String province = JQueryUtils.selectText(".city_province .city_list_tit", element)
|
||||
Elements cityElements = JQueryUtils.select(".CLICKDATA a", element)
|
||||
for (Element cityElement: cityElements) {
|
||||
String city = cityElement.text()
|
||||
if (citySet.contains(city)) {
|
||||
String url = cityElement.attr("href")
|
||||
String cityShort = StringUtils.substringBetween(url,"//", ".ke.com")
|
||||
DBObject dbObject = new BasicDBObject()
|
||||
dbObject.put("province", province)
|
||||
dbObject.put("city", city)
|
||||
dbObject.put("city_short", cityShort)
|
||||
dbObject.put("c_date", new Date())
|
||||
dbObject.put("m_date", new Date())
|
||||
dbObject.put("root", "ke.com")
|
||||
ret.add(dbObject)
|
||||
}
|
||||
}
|
||||
}
|
||||
cityConn.insert(ret)
|
||||
println("贝壳找房城市数据写入成功")
|
||||
}
|
||||
}
|
||||
}
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
package com.fangjia.script.temp.shenjian.util
|
||||
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
/**
|
||||
* @author shenjian
|
||||
* @since 2021/6/2
|
||||
*/
|
||||
class CityUtils {
|
||||
|
||||
static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017)
|
||||
|
||||
static listCitySet() {
|
||||
List<String> list = cityMapConn.distinct("fangjia_city")
|
||||
Set<String> citySet = new HashSet<>(list);
|
||||
return citySet
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user