修复名字丢失
This commit is contained in:
+129
-34
@@ -7,7 +7,6 @@ import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
@@ -18,10 +17,17 @@ import java.util.concurrent.atomic.AtomicLong
|
||||
class SeaweedSchoolLatLng {
|
||||
|
||||
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "seaweed_school", "192.168.0.235", 27018)
|
||||
static DBCollection school_collection = MongoUtils.getCollectionByName("fangjia_craw", "district_school", "192.168.0.235", 27018)
|
||||
|
||||
static ExecutorService executorService = Executors.newFixedThreadPool(20)
|
||||
|
||||
static void main(String[] args) {
|
||||
//latLng()
|
||||
updateAttribute()
|
||||
executorService.shutdown()
|
||||
}
|
||||
|
||||
static void updateAttribute() {
|
||||
AtomicLong atomicLong = new AtomicLong(0);
|
||||
collection.find(new BasicDBObject()).batchSize(1000).each {objT ->
|
||||
final DBObject obj = objT
|
||||
@@ -29,45 +35,68 @@ class SeaweedSchoolLatLng {
|
||||
@Override
|
||||
void run() {
|
||||
try {
|
||||
BasicDBList primarySchools = obj.primarySchool
|
||||
BasicDBList middleSchools = obj.middleSchool
|
||||
if (primarySchools != null) {
|
||||
for (BasicDBObject dbObject : primarySchools) {
|
||||
def name = dbObject.name
|
||||
if (dbObject.lng == null) {
|
||||
def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + name)
|
||||
if (location && location.lng) {
|
||||
def lng = Double.valueOf(location.lng)
|
||||
def lat = Double.valueOf(location.lat)
|
||||
dbObject.lng = lng
|
||||
dbObject.lat = lat
|
||||
def address = obj.address
|
||||
boolean update = false
|
||||
if (address.contains("浦东")) {
|
||||
BasicDBList primarySchools = obj.primarySchool
|
||||
BasicDBList middleSchools = obj.middleSchool
|
||||
DBObject schoolObj = school_collection.findOne(new BasicDBObject("city": '上海', 'district_name': obj.name))
|
||||
def quality = ''
|
||||
|
||||
if (primarySchools != null) {
|
||||
if (schoolObj && schoolObj.attribute) {
|
||||
quality = schoolObj.attribute
|
||||
}
|
||||
for (BasicDBObject dbObject : primarySchools) {
|
||||
if (dbObject.name == '第六师范学校第二附属小学西校区' || String.valueOf(dbObject.name).startsWith('第六师范附属小学')
|
||||
|| dbObject.name == '上海市第六师范第二附属小学七号桥校区' || dbObject.name == '康城学校康佳校区' || dbObject.name == '三墩学校') {
|
||||
dbObject.quality = '普通'
|
||||
update = true
|
||||
} else if (dbObject.name == '上海市实验学校东校(小学)' || dbObject.name == '浦东新区竹园小学沪东校区') {
|
||||
dbObject.quality = '市重点'
|
||||
update = true
|
||||
} else if (dbObject.name == '浦东新区上南二村小学南校区' || dbObject.name == '浦东新区新时代小学大华校区'
|
||||
|| dbObject.name == '浦东新区明珠小学A区') {
|
||||
dbObject.quality = '区重点'
|
||||
update = true
|
||||
}
|
||||
else if (quality != '' && schoolObj.category == '小学') {
|
||||
dbObject.quality = quality
|
||||
update = true
|
||||
} else {
|
||||
update = updateDbObjectQuality(dbObject);
|
||||
}
|
||||
}
|
||||
if (dbObject.lng != null) {
|
||||
String address = GaoDeUtils.getGaoDeAddress(dbObject.lng + "," + dbObject.lat)
|
||||
dbObject.address = address
|
||||
}
|
||||
}
|
||||
}
|
||||
if (middleSchools != null) {
|
||||
for (BasicDBObject dbObject : middleSchools) {
|
||||
def name = dbObject.name
|
||||
if (dbObject.lng == null) {
|
||||
def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + name)
|
||||
if (location && location.lng) {
|
||||
def lng = Double.valueOf(location.lng)
|
||||
def lat = Double.valueOf(location.lat)
|
||||
dbObject.lng = lng
|
||||
dbObject.lat = lat
|
||||
def update2 = false
|
||||
if (middleSchools != null) {
|
||||
for (BasicDBObject dbObject : middleSchools) {
|
||||
if (dbObject.name == '华东师范大学张江实验中学' || dbObject.name == '10上海市实验学校(初中)(全市招生)' || dbObject.name == '上海市实验学校附属光明学校总校区'
|
||||
|| dbObject.name == '上海市浦东新区进才实验中学南校' || String.valueOf(dbObject.name).startsWith('建平中学') || String.valueOf(dbObject.name).startsWith('建平实验中学')
|
||||
|| dbObject.name == '张江集团中学' || dbObject.name == '洋泾-菊园实验学校' || dbObject.name == '浦东外国语学校东校' || dbObject.name == '浦东新区建平中学南校' || String.valueOf(dbObject.name).startsWith('育人中学')
|
||||
|| dbObject.name == '致远中学' || String.valueOf(dbObject.name).startsWith('进才中学') || String.valueOf(dbObject.name).startsWith('进才实验中学')) {
|
||||
dbObject.quality = '市重点'
|
||||
update2 = true
|
||||
} else if (dbObject.name == '华东师范大学附属东昌中学南校潍坊校区' || dbObject.name == '下沙学校鹤沙路校区' || dbObject.name == '傅雷中学关岳路校区'
|
||||
|| dbObject.name == '傅雷中学瑞和路校区' || dbObject.name == '华东师范大学附属东昌中学南校张江校区' || dbObject.name == '南汇第一中学卫星东路校区'
|
||||
|| dbObject.name == '康城学校(拯安校区)(初中)' || dbObject.name == '建平香梅中学' || dbObject.name == '沪新中学' || dbObject.name == '浦东模范中学'
|
||||
|| dbObject.name == '浦东模范中学东校' || dbObject.name == '浦东模范实验中学' || String.valueOf(dbObject.name).startsWith('澧溪中学') ||
|
||||
dbObject.name == '清流中学' || String.valueOf(dbObject.name).startsWith('王港中学') || dbObject.name == '罗山中学'
|
||||
|| dbObject.name == '金杨中学' || dbObject.name == '香山中学' || dbObject.name == '高行中学') {
|
||||
dbObject.quality = '区重点'
|
||||
update2 = true
|
||||
} else if (quality != '' && schoolObj.category == '初中') {
|
||||
dbObject.quality = quality
|
||||
update2 = true
|
||||
} else {
|
||||
update2 = updateDbObjectQuality(dbObject)
|
||||
}
|
||||
}
|
||||
if (dbObject.lng != null) {
|
||||
String address = GaoDeUtils.getGaoDeAddress(dbObject.lng + "," + dbObject.lat)
|
||||
dbObject.address = address
|
||||
}
|
||||
}
|
||||
if (update || update2) {
|
||||
collection.update(new BasicDBObject('url', obj.url), new BasicDBObject('$set', obj), false, true)
|
||||
}
|
||||
}
|
||||
collection.update(new BasicDBObject('url', obj.url), new BasicDBObject('$set', obj), false, true)
|
||||
} catch(Exception ex) {
|
||||
ex.printStackTrace()
|
||||
} finally {
|
||||
@@ -81,6 +110,72 @@ class SeaweedSchoolLatLng {
|
||||
}
|
||||
})
|
||||
};
|
||||
executorService.shutdown()
|
||||
}
|
||||
static boolean updateDbObjectQuality(BasicDBObject dbObject) {
|
||||
if (dbObject.quality == '公办') {
|
||||
DBObject schoolObj = school_collection.findOne(new BasicDBObject("city": '上海', 'school_name': dbObject.name))
|
||||
if (schoolObj && schoolObj.attribute) {
|
||||
dbObject.quality = schoolObj.attribute
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
static void latLng() {
|
||||
AtomicLong atomicLong = new AtomicLong(0);
|
||||
collection.find(new BasicDBObject()).batchSize(1000).each {objT ->
|
||||
final DBObject obj = objT
|
||||
executorService.submit(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
try {
|
||||
def address = obj.address
|
||||
if (address.contains("浦东")) {
|
||||
BasicDBList primarySchools = obj.primarySchool
|
||||
BasicDBList middleSchools = obj.middleSchool
|
||||
if (primarySchools != null) {
|
||||
for (BasicDBObject dbObject : primarySchools) {
|
||||
def schoolAddress = dbObject.address
|
||||
if (dbObject.lng == null) {
|
||||
def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + schoolAddress)
|
||||
if (location && location.lng) {
|
||||
def lng = Double.valueOf(location.lng)
|
||||
def lat = Double.valueOf(location.lat)
|
||||
dbObject.lng = lng
|
||||
dbObject.lat = lat
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (middleSchools != null) {
|
||||
for (BasicDBObject dbObject : middleSchools) {
|
||||
def schoolAddress = dbObject.address
|
||||
if (dbObject.lng == null) {
|
||||
def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + schoolAddress)
|
||||
if (location && location.lng) {
|
||||
def lng = Double.valueOf(location.lng)
|
||||
def lat = Double.valueOf(location.lat)
|
||||
dbObject.lng = lng
|
||||
dbObject.lat = lat
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
collection.update(new BasicDBObject('url', obj.url), new BasicDBObject('$set', obj), false, true)
|
||||
}
|
||||
} catch(Exception ex) {
|
||||
ex.printStackTrace()
|
||||
} finally {
|
||||
atomicLong.incrementAndGet()
|
||||
synchronized (this) {
|
||||
if (atomicLong.longValue() % 100 == 0) {
|
||||
println(atomicLong.longValue())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
+1
-2
@@ -48,7 +48,6 @@ class SeaweedSchool {
|
||||
final int perPage = 10
|
||||
final int total = 4733;
|
||||
for (int page = 1; page < total / perPage; page++) {
|
||||
Thread.sleep(1000)
|
||||
print("当前页:" + page)
|
||||
try {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
@@ -71,7 +70,6 @@ class SeaweedSchool {
|
||||
List<HouseList.ContentDTO.ListDTO> listDTOS = houseList.getContent().getList();
|
||||
if (CollectionUtils.isNotEmpty(listDTOS)) {
|
||||
for (HouseList.ContentDTO.ListDTO listDTO : listDTOS) {
|
||||
Thread.sleep(500)
|
||||
Integer id = listDTO.getId();
|
||||
//# 房屋详情
|
||||
HouseDetail houseDetail = getHouseDetail(re, id, headers)
|
||||
@@ -146,6 +144,7 @@ class SeaweedSchool {
|
||||
schoolObject.put("town", contentDTO.getTown())
|
||||
schoolObject.put("distance", contentDTO.getDistance())
|
||||
schoolObject.put("address", contentDTO.getAddress())
|
||||
schoolObject.put("name", contentDTO.getName())
|
||||
schools.add(schoolObject)
|
||||
}
|
||||
return schools
|
||||
|
||||
Reference in New Issue
Block a user