修复名字丢失

This commit is contained in:
沈健
2021-09-05 13:52:50 +08:00
parent f267f393fe
commit 8d4c94431c
2 changed files with 130 additions and 36 deletions
@@ -7,7 +7,6 @@ import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
@@ -18,10 +17,17 @@ import java.util.concurrent.atomic.AtomicLong
class SeaweedSchoolLatLng {
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "seaweed_school", "192.168.0.235", 27018)
static DBCollection school_collection = MongoUtils.getCollectionByName("fangjia_craw", "district_school", "192.168.0.235", 27018)
static ExecutorService executorService = Executors.newFixedThreadPool(20)
static void main(String[] args) {
//latLng()
updateAttribute()
executorService.shutdown()
}
static void updateAttribute() {
AtomicLong atomicLong = new AtomicLong(0);
collection.find(new BasicDBObject()).batchSize(1000).each {objT ->
final DBObject obj = objT
@@ -29,45 +35,68 @@ class SeaweedSchoolLatLng {
@Override
void run() {
try {
BasicDBList primarySchools = obj.primarySchool
BasicDBList middleSchools = obj.middleSchool
if (primarySchools != null) {
for (BasicDBObject dbObject : primarySchools) {
def name = dbObject.name
if (dbObject.lng == null) {
def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + name)
if (location && location.lng) {
def lng = Double.valueOf(location.lng)
def lat = Double.valueOf(location.lat)
dbObject.lng = lng
dbObject.lat = lat
def address = obj.address
boolean update = false
if (address.contains("浦东")) {
BasicDBList primarySchools = obj.primarySchool
BasicDBList middleSchools = obj.middleSchool
DBObject schoolObj = school_collection.findOne(new BasicDBObject("city": '上海', 'district_name': obj.name))
def quality = ''
if (primarySchools != null) {
if (schoolObj && schoolObj.attribute) {
quality = schoolObj.attribute
}
for (BasicDBObject dbObject : primarySchools) {
if (dbObject.name == '第六师范学校第二附属小学西校区' || String.valueOf(dbObject.name).startsWith('第六师范附属小学')
|| dbObject.name == '上海市第六师范第二附属小学七号桥校区' || dbObject.name == '康城学校康佳校区' || dbObject.name == '三墩学校') {
dbObject.quality = '普通'
update = true
} else if (dbObject.name == '上海市实验学校东校(小学)' || dbObject.name == '浦东新区竹园小学沪东校区') {
dbObject.quality = '市重点'
update = true
} else if (dbObject.name == '浦东新区上南二村小学南校区' || dbObject.name == '浦东新区新时代小学大华校区'
|| dbObject.name == '浦东新区明珠小学A区') {
dbObject.quality = '区重点'
update = true
}
else if (quality != '' && schoolObj.category == '小学') {
dbObject.quality = quality
update = true
} else {
update = updateDbObjectQuality(dbObject);
}
}
if (dbObject.lng != null) {
String address = GaoDeUtils.getGaoDeAddress(dbObject.lng + "," + dbObject.lat)
dbObject.address = address
}
}
}
if (middleSchools != null) {
for (BasicDBObject dbObject : middleSchools) {
def name = dbObject.name
if (dbObject.lng == null) {
def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + name)
if (location && location.lng) {
def lng = Double.valueOf(location.lng)
def lat = Double.valueOf(location.lat)
dbObject.lng = lng
dbObject.lat = lat
def update2 = false
if (middleSchools != null) {
for (BasicDBObject dbObject : middleSchools) {
if (dbObject.name == '华东师范大学张江实验中学' || dbObject.name == '10上海市实验学校(初中)(全市招生)' || dbObject.name == '上海市实验学校附属光明学校总校区'
|| dbObject.name == '上海市浦东新区进才实验中学南校' || String.valueOf(dbObject.name).startsWith('建平中学') || String.valueOf(dbObject.name).startsWith('建平实验中学')
|| dbObject.name == '张江集团中学' || dbObject.name == '洋泾-菊园实验学校' || dbObject.name == '浦东外国语学校东校' || dbObject.name == '浦东新区建平中学南校' || String.valueOf(dbObject.name).startsWith('育人中学')
|| dbObject.name == '致远中学' || String.valueOf(dbObject.name).startsWith('进才中学') || String.valueOf(dbObject.name).startsWith('进才实验中学')) {
dbObject.quality = '市重点'
update2 = true
} else if (dbObject.name == '华东师范大学附属东昌中学南校潍坊校区' || dbObject.name == '下沙学校鹤沙路校区' || dbObject.name == '傅雷中学关岳路校区'
|| dbObject.name == '傅雷中学瑞和路校区' || dbObject.name == '华东师范大学附属东昌中学南校张江校区' || dbObject.name == '南汇第一中学卫星东路校区'
|| dbObject.name == '康城学校(拯安校区)(初中)' || dbObject.name == '建平香梅中学' || dbObject.name == '沪新中学' || dbObject.name == '浦东模范中学'
|| dbObject.name == '浦东模范中学东校' || dbObject.name == '浦东模范实验中学' || String.valueOf(dbObject.name).startsWith('澧溪中学') ||
dbObject.name == '清流中学' || String.valueOf(dbObject.name).startsWith('王港中学') || dbObject.name == '罗山中学'
|| dbObject.name == '金杨中学' || dbObject.name == '香山中学' || dbObject.name == '高行中学') {
dbObject.quality = '区重点'
update2 = true
} else if (quality != '' && schoolObj.category == '初中') {
dbObject.quality = quality
update2 = true
} else {
update2 = updateDbObjectQuality(dbObject)
}
}
if (dbObject.lng != null) {
String address = GaoDeUtils.getGaoDeAddress(dbObject.lng + "," + dbObject.lat)
dbObject.address = address
}
}
if (update || update2) {
collection.update(new BasicDBObject('url', obj.url), new BasicDBObject('$set', obj), false, true)
}
}
collection.update(new BasicDBObject('url', obj.url), new BasicDBObject('$set', obj), false, true)
} catch(Exception ex) {
ex.printStackTrace()
} finally {
@@ -81,6 +110,72 @@ class SeaweedSchoolLatLng {
}
})
};
executorService.shutdown()
}
static boolean updateDbObjectQuality(BasicDBObject dbObject) {
if (dbObject.quality == '公办') {
DBObject schoolObj = school_collection.findOne(new BasicDBObject("city": '上海', 'school_name': dbObject.name))
if (schoolObj && schoolObj.attribute) {
dbObject.quality = schoolObj.attribute
return true
}
}
return false
}
static void latLng() {
AtomicLong atomicLong = new AtomicLong(0);
collection.find(new BasicDBObject()).batchSize(1000).each {objT ->
final DBObject obj = objT
executorService.submit(new Runnable() {
@Override
void run() {
try {
def address = obj.address
if (address.contains("浦东")) {
BasicDBList primarySchools = obj.primarySchool
BasicDBList middleSchools = obj.middleSchool
if (primarySchools != null) {
for (BasicDBObject dbObject : primarySchools) {
def schoolAddress = dbObject.address
if (dbObject.lng == null) {
def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + schoolAddress)
if (location && location.lng) {
def lng = Double.valueOf(location.lng)
def lat = Double.valueOf(location.lat)
dbObject.lng = lng
dbObject.lat = lat
}
}
}
}
if (middleSchools != null) {
for (BasicDBObject dbObject : middleSchools) {
def schoolAddress = dbObject.address
if (dbObject.lng == null) {
def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + schoolAddress)
if (location && location.lng) {
def lng = Double.valueOf(location.lng)
def lat = Double.valueOf(location.lat)
dbObject.lng = lng
dbObject.lat = lat
}
}
}
}
collection.update(new BasicDBObject('url', obj.url), new BasicDBObject('$set', obj), false, true)
}
} catch(Exception ex) {
ex.printStackTrace()
} finally {
atomicLong.incrementAndGet()
synchronized (this) {
if (atomicLong.longValue() % 100 == 0) {
println(atomicLong.longValue())
}
}
}
}
})
};
}
}
@@ -48,7 +48,6 @@ class SeaweedSchool {
final int perPage = 10
final int total = 4733;
for (int page = 1; page < total / perPage; page++) {
Thread.sleep(1000)
print("当前页:" + page)
try {
Map<String, String> params = new HashMap<String, String>();
@@ -71,7 +70,6 @@ class SeaweedSchool {
List<HouseList.ContentDTO.ListDTO> listDTOS = houseList.getContent().getList();
if (CollectionUtils.isNotEmpty(listDTOS)) {
for (HouseList.ContentDTO.ListDTO listDTO : listDTOS) {
Thread.sleep(500)
Integer id = listDTO.getId();
//# 房屋详情
HouseDetail houseDetail = getHouseDetail(re, id, headers)
@@ -146,6 +144,7 @@ class SeaweedSchool {
schoolObject.put("town", contentDTO.getTown())
schoolObject.put("distance", contentDTO.getDistance())
schoolObject.put("address", contentDTO.getAddress())
schoolObject.put("name", contentDTO.getName())
schools.add(schoolObject)
}
return schools