新增批量写入方法
This commit is contained in:
+60
-46
@@ -13,7 +13,13 @@ import com.fangjia.script.wenyan.auto.KeywordSearchUtils
|
|||||||
import com.mongodb.BasicDBObject
|
import com.mongodb.BasicDBObject
|
||||||
import com.mongodb.DBCollection
|
import com.mongodb.DBCollection
|
||||||
import com.mongodb.DBObject
|
import com.mongodb.DBObject
|
||||||
|
import com.mongodb.client.model.Filters
|
||||||
|
import com.mongodb.client.model.UpdateOneModel
|
||||||
|
import com.mongodb.client.model.UpdateOptions
|
||||||
|
import com.mongodb.client.model.WriteModel
|
||||||
import org.apache.commons.collections.CollectionUtils
|
import org.apache.commons.collections.CollectionUtils
|
||||||
|
import org.bson.Document
|
||||||
|
import org.bson.conversions.Bson
|
||||||
import org.bson.types.ObjectId
|
import org.bson.types.ObjectId
|
||||||
|
|
||||||
import java.util.concurrent.ExecutorService
|
import java.util.concurrent.ExecutorService
|
||||||
@@ -27,74 +33,82 @@ import java.util.concurrent.Executors
|
|||||||
*/
|
*/
|
||||||
class SeaweedAroundPoiSingal {
|
class SeaweedAroundPoiSingal {
|
||||||
|
|
||||||
static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017)
|
static def conn136 = MongoUtils.getCollectionByName3("fangjia", "seaweed_poi_sync", "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017")
|
||||||
static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi"
|
static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi"
|
||||||
|
|
||||||
static void main(String[] args) {
|
static void main(String[] args) {
|
||||||
def types = listAllType()
|
getNewSeaweed()
|
||||||
//def types = "0601"
|
// def types = listAllType()
|
||||||
DBObject query = new BasicDBObject()
|
// //def types = "0601"
|
||||||
query.put("status", 1)
|
// DBObject query = new BasicDBObject()
|
||||||
conn136.find(query).batchSize(30).each {objTemp ->
|
// query.put("status", 1)
|
||||||
final def obj = objTemp
|
// conn136.find(query).batchSize(30).each {objTemp ->
|
||||||
def city = obj.city
|
// final def obj = objTemp
|
||||||
def lng = obj.lng
|
// def city = obj.city
|
||||||
def lat = obj.lat
|
// def lng = obj.lng
|
||||||
List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "")
|
// def lat = obj.lat
|
||||||
if (CollectionUtils.isNotEmpty(pois)) {
|
// List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "")
|
||||||
String jsonStr = JSONObject.toJSONString(pois)
|
// if (CollectionUtils.isNotEmpty(pois)) {
|
||||||
HttpUtils.postForJson(URL, jsonStr)
|
// String jsonStr = JSONObject.toJSONString(pois)
|
||||||
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI")))
|
// HttpUtils.postForJson(URL, jsonStr)
|
||||||
} else {
|
// conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI")))
|
||||||
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空")))
|
// } else {
|
||||||
}
|
// conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空")))
|
||||||
println("已处理小区:" + obj)
|
// }
|
||||||
}
|
// println("已处理小区:" + obj)
|
||||||
|
// }
|
||||||
}
|
}
|
||||||
|
|
||||||
static getNewSeaweed() {
|
static getNewSeaweed() {
|
||||||
DBObject query = new BasicDBObject()
|
DBObject query = new BasicDBObject()
|
||||||
query.put("cat", "district")
|
query.put("cat", "district")
|
||||||
query.put("visible", 0)
|
query.put("visible", 0)
|
||||||
//query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a"))
|
|
||||||
|
|
||||||
DBObject filter = new BasicDBObject()
|
DBObject filter = new BasicDBObject()
|
||||||
filter.put("_id", 1)
|
filter.put("_id", 1)
|
||||||
filter.put("city", 1)
|
filter.put("city", 1)
|
||||||
filter.put("region", 1)
|
filter.put("region", 1)
|
||||||
filter.put("name", 1)
|
filter.put("name", 1)
|
||||||
|
filter.put("alias", 1)
|
||||||
|
filter.put("address", 1)
|
||||||
|
filter.put("addr_alias", 1)
|
||||||
filter.put("lng", 1)
|
filter.put("lng", 1)
|
||||||
filter.put("lat", 1)
|
filter.put("lat", 1)
|
||||||
|
|
||||||
List list = []
|
List<WriteModel<Document>> writeModelList = new ArrayList<>()
|
||||||
DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj ->
|
DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj ->
|
||||||
if (conn136.count(new BasicDBObject("_id", obj._id)) > 0) {
|
Document updateBson = new Document()
|
||||||
// 已存在小区,无需同步
|
Document update = new Document()
|
||||||
} else {
|
update.put("city", obj.city)
|
||||||
DBObject newObj = new BasicDBObject()
|
update.put("region", obj.region)
|
||||||
newObj.put("_id", obj._id)
|
update.put("name", obj.name)
|
||||||
newObj.put("city", obj.city)
|
update.put("alias", obj.alias)
|
||||||
newObj.put("region", obj.region)
|
update.put("address", obj.address)
|
||||||
newObj.put("name", obj.name)
|
update.put("addr_alias", obj.addr_alias)
|
||||||
try {
|
try {
|
||||||
newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng))
|
update.put("lng", SeaweedUtils.formatLatLng(obj.lng))
|
||||||
newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat))
|
update.put("lat", SeaweedUtils.formatLatLng(obj.lat))
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
print("经纬度为空:" + obj._id + " " + obj)
|
print("经纬度为空:" + obj._id + " " + obj)
|
||||||
ex.printStackTrace()
|
}
|
||||||
}
|
update.put("status", 1)
|
||||||
newObj.put("status", 1)
|
update.put("flag", "未精确获取高德信息")
|
||||||
newObj.put("flag", "未获取周边POI")
|
updateBson.append('$set', update)
|
||||||
|
|
||||||
list << newObj
|
Bson filterBson = Filters.eq("_id", obj._id)
|
||||||
if (list.size() == 5000) {
|
UpdateOptions updateOptions = new UpdateOptions();
|
||||||
conn136.insert(list)
|
updateOptions.upsert(true)
|
||||||
list.clear()
|
UpdateOneModel updateOneModel = new UpdateOneModel(filterBson, updateBson, updateOptions);
|
||||||
}
|
writeModelList.add(updateOneModel)
|
||||||
|
if (writeModelList.size() == 5000) {
|
||||||
|
conn136.bulkWrite(writeModelList)
|
||||||
|
writeModelList.clear()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
conn136.insert(list)
|
if (writeModelList.size() > 0) {
|
||||||
list.clear()
|
conn136.bulkWrite(writeModelList)
|
||||||
|
writeModelList.clear()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static def getByIdAndInsert(String id) {
|
static def getByIdAndInsert(String id) {
|
||||||
|
|||||||
Reference in New Issue
Block a user