新增58小区总量获取

This commit is contained in:
沈健
2021-05-26 11:57:30 +08:00
parent 0aa41d570a
commit bc74173447
6 changed files with 194 additions and 1 deletions
@@ -0,0 +1,65 @@
package com.fangjia.script.temp.shenjian.crawl
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.crawl.InvokeHttpRequest
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
/**
* 统计58小区总量
*
* @author shenjian
* @since 2021/5/26
*/
class Count58Seaweed {
static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018)
static void main(String[] args) {
crawlSeaweedCount()
println(count())
}
static crawlSeaweedCount() {
cityConn.find(new BasicDBObject("root", "58.com").append("district_count", new BasicDBObject('$eq', null)),
new BasicDBObject("_id", 1).append("city_short", 1).append("city", 1))
.batchSize(100).each { obj ->
def id = obj._id
def city_short = obj.city_short
final def URL = "https://" + city_short + ".58.com/xiaoqu/"
try {
String res = ABuYunUtil.getReponseContent2(URL, "utf-8")
if (res) {
Document doc = Jsoup.parse(res);
String str = JQueryUtils.selectText(".total-info", doc)
// 117.60.236.41
if (str) {
str = str.replace("共找到", "").replace("个小区", "").trim()
int count = Integer.parseInt(str)
cityConn.update(new BasicDBObject("_id", id), new BasicDBObject('$set', new BasicDBObject("district_count", count)))
println("当前处理城市" + obj.city)
}
}
} catch(Exception ex) {
ex.printStackTrace()
}
}
}
static count() {
int number = 0;
cityConn.find(new BasicDBObject("root", "58.com"), new BasicDBObject("district_count", 1))
.batchSize(100).each { obj ->
if (obj.district_count) {
number += obj.district_count
}
}
return number
}
}
@@ -29,7 +29,7 @@ class SeaweedAroundPoi {
static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017)
static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi"
static ExecutorService executorService = Executors.newFixedThreadPool(30)
static ExecutorService executorService = Executors.newFixedThreadPool(10)
static void main(String[] args) {
def types = listAllType()
@@ -0,0 +1,128 @@
package com.fangjia.script.temp.shenjian.poi
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.wenyan.auto.KeywordSearchUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.collections.CollectionUtils
import org.bson.types.ObjectId
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 小区周边POI数据获取单线程方式
*
* @author shenjian
* @since 2021/5/24
*/
class SeaweedAroundPoiSingal {
static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017)
static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi"
static void main(String[] args) {
def types = listAllType()
//def types = "0601"
DBObject query = new BasicDBObject()
query.put("status", 1)
conn136.find(query).batchSize(30).each {objTemp ->
final def obj = objTemp
def city = obj.city
def lng = obj.lng
def lat = obj.lat
List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "")
if (CollectionUtils.isNotEmpty(pois)) {
String jsonStr = JSONObject.toJSONString(pois)
HttpUtils.postForJson(URL, jsonStr)
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI")))
} else {
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空")))
}
println("已处理小区:" + obj)
}
}
static getNewSeaweed() {
DBObject query = new BasicDBObject()
query.put("cat", "district")
query.put("visible", 0)
//query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a"))
DBObject filter = new BasicDBObject()
filter.put("_id", 1)
filter.put("city", 1)
filter.put("region", 1)
filter.put("name", 1)
filter.put("lng", 1)
filter.put("lat", 1)
List list = []
DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj ->
if (conn136.count(new BasicDBObject("_id", obj._id)) > 0) {
// 已存在小区,无需同步
} else {
DBObject newObj = new BasicDBObject()
newObj.put("_id", obj._id)
newObj.put("city", obj.city)
newObj.put("region", obj.region)
newObj.put("name", obj.name)
try {
newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng))
newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat))
} catch (Exception ex) {
print("经纬度为空:" + obj._id + " " + obj)
ex.printStackTrace()
}
newObj.put("status", 1)
newObj.put("flag", "未获取周边POI")
list << newObj
if (list.size() == 5000) {
conn136.insert(list)
list.clear()
}
}
}
conn136.insert(list)
list.clear()
}
static def getByIdAndInsert(String id) {
BasicDBObject obj = DbUtils.seaweed.findById(id);
DBObject newObj = new BasicDBObject()
newObj.put("_id", obj._id)
newObj.put("city", obj.city)
newObj.put("region", obj.region)
newObj.put("name", obj.name)
try {
newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng))
newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat))
} catch (Exception ex) {
print("经纬度为空:" + obj._id + " " + obj)
ex.printStackTrace()
}
newObj.put("status", 1)
newObj.put("flag", "未获取周边POI")
conn136.insert(newObj)
}
static def listAllType() {
StringBuilder ret = new StringBuilder()
for (def val : KeywordSearchUtils.KEYWORDS_MAP.values()) {
ret.append(val).append("|")
}
String result = ret.toString()
return result.substring(0, result.length() - 1)
}
}