新增58小区总量获取
This commit is contained in:
+49
@@ -0,0 +1,49 @@
|
||||
package com.fangjia.script.temp.shenjian.crawl
|
||||
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.script.crawl.InvokeHttpRequest
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.jsoup.Jsoup
|
||||
import org.jsoup.nodes.Document
|
||||
|
||||
/**
|
||||
* 统计58小区总量
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2021/5/26
|
||||
*/
|
||||
class Count58Seaweed {
|
||||
|
||||
static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018)
|
||||
|
||||
static void main(String[] args) {
|
||||
cityConn.find(new BasicDBObject("root", "58.com").append("district_count", new BasicDBObject('$eq', null)),
|
||||
new BasicDBObject("_id", 1).append("city_short", 1).append("city", 1))
|
||||
.batchSize(100).each { obj ->
|
||||
def id = obj._id
|
||||
def city_short = obj.city_short
|
||||
final def URL = "https://" + city_short + ".58.com/xiaoqu/"
|
||||
try {
|
||||
String res = ABuYunUtil.getReponseContent2(URL, "utf-8")
|
||||
if (res) {
|
||||
Document doc = Jsoup.parse(res);
|
||||
String str = JQueryUtils.selectText(".total-info", doc)
|
||||
// 117.60.236.41
|
||||
if (str) {
|
||||
str = str.replace("共找到", "").replace("个小区", "").trim()
|
||||
int count = Integer.parseInt(str)
|
||||
cityConn.update(new BasicDBObject("_id", id), new BasicDBObject('$set', new BasicDBObject("district_count", count)))
|
||||
println("当前处理城市" + obj.city)
|
||||
}
|
||||
}
|
||||
} catch(Exception ex) {
|
||||
ex.printStackTrace()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -29,7 +29,7 @@ class SeaweedAroundPoi {
|
||||
|
||||
static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017)
|
||||
static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi"
|
||||
static ExecutorService executorService = Executors.newFixedThreadPool(30)
|
||||
static ExecutorService executorService = Executors.newFixedThreadPool(10)
|
||||
|
||||
static void main(String[] args) {
|
||||
def types = listAllType()
|
||||
|
||||
+128
@@ -0,0 +1,128 @@
|
||||
package com.fangjia.script.temp.shenjian.poi
|
||||
|
||||
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.statistics.poi.AmapPoi
|
||||
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
|
||||
import com.fangjia.script.temp.shenjian.util.HttpUtils
|
||||
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.wenyan.auto.KeywordSearchUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.commons.collections.CollectionUtils
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
|
||||
/**
|
||||
* 小区周边POI数据获取单线程方式
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2021/5/24
|
||||
*/
|
||||
class SeaweedAroundPoiSingal {
|
||||
|
||||
static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017)
|
||||
static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi"
|
||||
|
||||
static void main(String[] args) {
|
||||
def types = listAllType()
|
||||
//def types = "0601"
|
||||
DBObject query = new BasicDBObject()
|
||||
query.put("status", 1)
|
||||
conn136.find(query).batchSize(30).each {objTemp ->
|
||||
final def obj = objTemp
|
||||
def city = obj.city
|
||||
def lng = obj.lng
|
||||
def lat = obj.lat
|
||||
List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "")
|
||||
if (CollectionUtils.isNotEmpty(pois)) {
|
||||
String jsonStr = JSONObject.toJSONString(pois)
|
||||
HttpUtils.postForJson(URL, jsonStr)
|
||||
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI")))
|
||||
} else {
|
||||
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空")))
|
||||
}
|
||||
println("已处理小区:" + obj)
|
||||
}
|
||||
}
|
||||
|
||||
static getNewSeaweed() {
|
||||
DBObject query = new BasicDBObject()
|
||||
query.put("cat", "district")
|
||||
query.put("visible", 0)
|
||||
//query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a"))
|
||||
|
||||
DBObject filter = new BasicDBObject()
|
||||
filter.put("_id", 1)
|
||||
filter.put("city", 1)
|
||||
filter.put("region", 1)
|
||||
filter.put("name", 1)
|
||||
filter.put("lng", 1)
|
||||
filter.put("lat", 1)
|
||||
|
||||
List list = []
|
||||
DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj ->
|
||||
if (conn136.count(new BasicDBObject("_id", obj._id)) > 0) {
|
||||
// 已存在小区,无需同步
|
||||
} else {
|
||||
DBObject newObj = new BasicDBObject()
|
||||
newObj.put("_id", obj._id)
|
||||
newObj.put("city", obj.city)
|
||||
newObj.put("region", obj.region)
|
||||
newObj.put("name", obj.name)
|
||||
try {
|
||||
newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng))
|
||||
newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat))
|
||||
} catch (Exception ex) {
|
||||
print("经纬度为空:" + obj._id + " " + obj)
|
||||
ex.printStackTrace()
|
||||
}
|
||||
newObj.put("status", 1)
|
||||
newObj.put("flag", "未获取周边POI")
|
||||
|
||||
list << newObj
|
||||
if (list.size() == 5000) {
|
||||
conn136.insert(list)
|
||||
list.clear()
|
||||
}
|
||||
}
|
||||
}
|
||||
conn136.insert(list)
|
||||
list.clear()
|
||||
}
|
||||
|
||||
static def getByIdAndInsert(String id) {
|
||||
BasicDBObject obj = DbUtils.seaweed.findById(id);
|
||||
DBObject newObj = new BasicDBObject()
|
||||
newObj.put("_id", obj._id)
|
||||
newObj.put("city", obj.city)
|
||||
newObj.put("region", obj.region)
|
||||
newObj.put("name", obj.name)
|
||||
try {
|
||||
newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng))
|
||||
newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat))
|
||||
} catch (Exception ex) {
|
||||
print("经纬度为空:" + obj._id + " " + obj)
|
||||
ex.printStackTrace()
|
||||
}
|
||||
newObj.put("status", 1)
|
||||
newObj.put("flag", "未获取周边POI")
|
||||
|
||||
conn136.insert(newObj)
|
||||
}
|
||||
|
||||
static def listAllType() {
|
||||
StringBuilder ret = new StringBuilder()
|
||||
for (def val : KeywordSearchUtils.KEYWORDS_MAP.values()) {
|
||||
ret.append(val).append("|")
|
||||
}
|
||||
String result = ret.toString()
|
||||
return result.substring(0, result.length() - 1)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user