新增批量写入方法

This commit is contained in:
Jian Shen
2022-05-25 14:05:55 +08:00
parent a9e7c4fa4b
commit 41b0b31b75
@@ -13,7 +13,13 @@ import com.fangjia.script.wenyan.auto.KeywordSearchUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import com.mongodb.DBObject import com.mongodb.DBObject
import com.mongodb.client.model.Filters
import com.mongodb.client.model.UpdateOneModel
import com.mongodb.client.model.UpdateOptions
import com.mongodb.client.model.WriteModel
import org.apache.commons.collections.CollectionUtils import org.apache.commons.collections.CollectionUtils
import org.bson.Document
import org.bson.conversions.Bson
import org.bson.types.ObjectId import org.bson.types.ObjectId
import java.util.concurrent.ExecutorService import java.util.concurrent.ExecutorService
@@ -27,74 +33,82 @@ import java.util.concurrent.Executors
*/ */
class SeaweedAroundPoiSingal { class SeaweedAroundPoiSingal {
static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) static def conn136 = MongoUtils.getCollectionByName3("fangjia", "seaweed_poi_sync", "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017")
static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi" static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi"
static void main(String[] args) { static void main(String[] args) {
def types = listAllType() getNewSeaweed()
//def types = "0601" // def types = listAllType()
DBObject query = new BasicDBObject() // //def types = "0601"
query.put("status", 1) // DBObject query = new BasicDBObject()
conn136.find(query).batchSize(30).each {objTemp -> // query.put("status", 1)
final def obj = objTemp // conn136.find(query).batchSize(30).each {objTemp ->
def city = obj.city // final def obj = objTemp
def lng = obj.lng // def city = obj.city
def lat = obj.lat // def lng = obj.lng
List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") // def lat = obj.lat
if (CollectionUtils.isNotEmpty(pois)) { // List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "")
String jsonStr = JSONObject.toJSONString(pois) // if (CollectionUtils.isNotEmpty(pois)) {
HttpUtils.postForJson(URL, jsonStr) // String jsonStr = JSONObject.toJSONString(pois)
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) // HttpUtils.postForJson(URL, jsonStr)
} else { // conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI")))
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空"))) // } else {
} // conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空")))
println("已处理小区:" + obj) // }
} // println("已处理小区:" + obj)
// }
} }
static getNewSeaweed() { static getNewSeaweed() {
DBObject query = new BasicDBObject() DBObject query = new BasicDBObject()
query.put("cat", "district") query.put("cat", "district")
query.put("visible", 0) query.put("visible", 0)
//query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a"))
DBObject filter = new BasicDBObject() DBObject filter = new BasicDBObject()
filter.put("_id", 1) filter.put("_id", 1)
filter.put("city", 1) filter.put("city", 1)
filter.put("region", 1) filter.put("region", 1)
filter.put("name", 1) filter.put("name", 1)
filter.put("alias", 1)
filter.put("address", 1)
filter.put("addr_alias", 1)
filter.put("lng", 1) filter.put("lng", 1)
filter.put("lat", 1) filter.put("lat", 1)
List list = [] List<WriteModel<Document>> writeModelList = new ArrayList<>()
DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj -> DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj ->
if (conn136.count(new BasicDBObject("_id", obj._id)) > 0) { Document updateBson = new Document()
// 已存在小区,无需同步 Document update = new Document()
} else { update.put("city", obj.city)
DBObject newObj = new BasicDBObject() update.put("region", obj.region)
newObj.put("_id", obj._id) update.put("name", obj.name)
newObj.put("city", obj.city) update.put("alias", obj.alias)
newObj.put("region", obj.region) update.put("address", obj.address)
newObj.put("name", obj.name) update.put("addr_alias", obj.addr_alias)
try { try {
newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) update.put("lng", SeaweedUtils.formatLatLng(obj.lng))
newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) update.put("lat", SeaweedUtils.formatLatLng(obj.lat))
} catch (Exception ex) { } catch (Exception ex) {
print("经纬度为空:" + obj._id + " " + obj) print("经纬度为空:" + obj._id + " " + obj)
ex.printStackTrace() }
} update.put("status", 1)
newObj.put("status", 1) update.put("flag", "未精确获取高德信息")
newObj.put("flag", "未获取周边POI") updateBson.append('$set', update)
list << newObj Bson filterBson = Filters.eq("_id", obj._id)
if (list.size() == 5000) { UpdateOptions updateOptions = new UpdateOptions();
conn136.insert(list) updateOptions.upsert(true)
list.clear() UpdateOneModel updateOneModel = new UpdateOneModel(filterBson, updateBson, updateOptions);
} writeModelList.add(updateOneModel)
if (writeModelList.size() == 5000) {
conn136.bulkWrite(writeModelList)
writeModelList.clear()
} }
} }
conn136.insert(list) if (writeModelList.size() > 0) {
list.clear() conn136.bulkWrite(writeModelList)
writeModelList.clear()
}
} }
static def getByIdAndInsert(String id) { static def getByIdAndInsert(String id) {