diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedSchoolLatLng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedSchoolLatLng.groovy index d511bb745..ad1676b7d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedSchoolLatLng.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedSchoolLatLng.groovy @@ -7,7 +7,6 @@ import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject -import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong @@ -18,10 +17,17 @@ import java.util.concurrent.atomic.AtomicLong class SeaweedSchoolLatLng { static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "seaweed_school", "192.168.0.235", 27018) + static DBCollection school_collection = MongoUtils.getCollectionByName("fangjia_craw", "district_school", "192.168.0.235", 27018) static ExecutorService executorService = Executors.newFixedThreadPool(20) static void main(String[] args) { + //latLng() + updateAttribute() + executorService.shutdown() + } + + static void updateAttribute() { AtomicLong atomicLong = new AtomicLong(0); collection.find(new BasicDBObject()).batchSize(1000).each {objT -> final DBObject obj = objT @@ -29,45 +35,68 @@ class SeaweedSchoolLatLng { @Override void run() { try { - BasicDBList primarySchools = obj.primarySchool - BasicDBList middleSchools = obj.middleSchool - if (primarySchools != null) { - for (BasicDBObject dbObject : primarySchools) { - def name = dbObject.name - if (dbObject.lng == null) { - def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + name) - if (location && location.lng) { - def lng = Double.valueOf(location.lng) - def lat = Double.valueOf(location.lat) - dbObject.lng = lng - dbObject.lat = lat + def address = obj.address + boolean update = false + if (address.contains("浦东")) { + BasicDBList primarySchools = obj.primarySchool + BasicDBList middleSchools = obj.middleSchool + DBObject schoolObj = school_collection.findOne(new BasicDBObject("city": '上海', 'district_name': obj.name)) + def quality = '' + + if (primarySchools != null) { + if (schoolObj && schoolObj.attribute) { + quality = schoolObj.attribute + } + for (BasicDBObject dbObject : primarySchools) { + if (dbObject.name == '第六师范学校第二附属小学西校区' || String.valueOf(dbObject.name).startsWith('第六师范附属小学') + || dbObject.name == '上海市第六师范第二附属小学七号桥校区' || dbObject.name == '康城学校康佳校区' || dbObject.name == '三墩学校') { + dbObject.quality = '普通' + update = true + } else if (dbObject.name == '上海市实验学校东校(小学)' || dbObject.name == '浦东新区竹园小学沪东校区') { + dbObject.quality = '市重点' + update = true + } else if (dbObject.name == '浦东新区上南二村小学南校区' || dbObject.name == '浦东新区新时代小学大华校区' + || dbObject.name == '浦东新区明珠小学A区') { + dbObject.quality = '区重点' + update = true + } + else if (quality != '' && schoolObj.category == '小学') { + dbObject.quality = quality + update = true + } else { + update = updateDbObjectQuality(dbObject); } } - if (dbObject.lng != null) { - String address = GaoDeUtils.getGaoDeAddress(dbObject.lng + "," + dbObject.lat) - dbObject.address = address - } } - } - if (middleSchools != null) { - for (BasicDBObject dbObject : middleSchools) { - def name = dbObject.name - if (dbObject.lng == null) { - def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + name) - if (location && location.lng) { - def lng = Double.valueOf(location.lng) - def lat = Double.valueOf(location.lat) - dbObject.lng = lng - dbObject.lat = lat + def update2 = false + if (middleSchools != null) { + for (BasicDBObject dbObject : middleSchools) { + if (dbObject.name == '华东师范大学张江实验中学' || dbObject.name == '10上海市实验学校(初中)(全市招生)' || dbObject.name == '上海市实验学校附属光明学校总校区' + || dbObject.name == '上海市浦东新区进才实验中学南校' || String.valueOf(dbObject.name).startsWith('建平中学') || String.valueOf(dbObject.name).startsWith('建平实验中学') + || dbObject.name == '张江集团中学' || dbObject.name == '洋泾-菊园实验学校' || dbObject.name == '浦东外国语学校东校' || dbObject.name == '浦东新区建平中学南校' || String.valueOf(dbObject.name).startsWith('育人中学') + || dbObject.name == '致远中学' || String.valueOf(dbObject.name).startsWith('进才中学') || String.valueOf(dbObject.name).startsWith('进才实验中学')) { + dbObject.quality = '市重点' + update2 = true + } else if (dbObject.name == '华东师范大学附属东昌中学南校潍坊校区' || dbObject.name == '下沙学校鹤沙路校区' || dbObject.name == '傅雷中学关岳路校区' + || dbObject.name == '傅雷中学瑞和路校区' || dbObject.name == '华东师范大学附属东昌中学南校张江校区' || dbObject.name == '南汇第一中学卫星东路校区' + || dbObject.name == '康城学校(拯安校区)(初中)' || dbObject.name == '建平香梅中学' || dbObject.name == '沪新中学' || dbObject.name == '浦东模范中学' + || dbObject.name == '浦东模范中学东校' || dbObject.name == '浦东模范实验中学' || String.valueOf(dbObject.name).startsWith('澧溪中学') || + dbObject.name == '清流中学' || String.valueOf(dbObject.name).startsWith('王港中学') || dbObject.name == '罗山中学' + || dbObject.name == '金杨中学' || dbObject.name == '香山中学' || dbObject.name == '高行中学') { + dbObject.quality = '区重点' + update2 = true + } else if (quality != '' && schoolObj.category == '初中') { + dbObject.quality = quality + update2 = true + } else { + update2 = updateDbObjectQuality(dbObject) } } - if (dbObject.lng != null) { - String address = GaoDeUtils.getGaoDeAddress(dbObject.lng + "," + dbObject.lat) - dbObject.address = address - } + } + if (update || update2) { + collection.update(new BasicDBObject('url', obj.url), new BasicDBObject('$set', obj), false, true) } } - collection.update(new BasicDBObject('url', obj.url), new BasicDBObject('$set', obj), false, true) } catch(Exception ex) { ex.printStackTrace() } finally { @@ -81,6 +110,72 @@ class SeaweedSchoolLatLng { } }) }; - executorService.shutdown() + } + static boolean updateDbObjectQuality(BasicDBObject dbObject) { + if (dbObject.quality == '公办') { + DBObject schoolObj = school_collection.findOne(new BasicDBObject("city": '上海', 'school_name': dbObject.name)) + if (schoolObj && schoolObj.attribute) { + dbObject.quality = schoolObj.attribute + return true + } + } + return false + } + + static void latLng() { + AtomicLong atomicLong = new AtomicLong(0); + collection.find(new BasicDBObject()).batchSize(1000).each {objT -> + final DBObject obj = objT + executorService.submit(new Runnable() { + @Override + void run() { + try { + def address = obj.address + if (address.contains("浦东")) { + BasicDBList primarySchools = obj.primarySchool + BasicDBList middleSchools = obj.middleSchool + if (primarySchools != null) { + for (BasicDBObject dbObject : primarySchools) { + def schoolAddress = dbObject.address + if (dbObject.lng == null) { + def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + schoolAddress) + if (location && location.lng) { + def lng = Double.valueOf(location.lng) + def lat = Double.valueOf(location.lat) + dbObject.lng = lng + dbObject.lat = lat + } + } + } + } + if (middleSchools != null) { + for (BasicDBObject dbObject : middleSchools) { + def schoolAddress = dbObject.address + if (dbObject.lng == null) { + def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + schoolAddress) + if (location && location.lng) { + def lng = Double.valueOf(location.lng) + def lat = Double.valueOf(location.lat) + dbObject.lng = lng + dbObject.lat = lat + } + } + } + } + collection.update(new BasicDBObject('url', obj.url), new BasicDBObject('$set', obj), false, true) + } + } catch(Exception ex) { + ex.printStackTrace() + } finally { + atomicLong.incrementAndGet() + synchronized (this) { + if (atomicLong.longValue() % 100 == 0) { + println(atomicLong.longValue()) + } + } + } + } + }) + }; } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy index b988b6a9d..b3ab13fef 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy @@ -48,7 +48,6 @@ class SeaweedSchool { final int perPage = 10 final int total = 4733; for (int page = 1; page < total / perPage; page++) { - Thread.sleep(1000) print("当前页:" + page) try { Map params = new HashMap(); @@ -71,7 +70,6 @@ class SeaweedSchool { List listDTOS = houseList.getContent().getList(); if (CollectionUtils.isNotEmpty(listDTOS)) { for (HouseList.ContentDTO.ListDTO listDTO : listDTOS) { - Thread.sleep(500) Integer id = listDTO.getId(); //# 房屋详情 HouseDetail houseDetail = getHouseDetail(re, id, headers) @@ -146,6 +144,7 @@ class SeaweedSchool { schoolObject.put("town", contentDTO.getTown()) schoolObject.put("distance", contentDTO.getDistance()) schoolObject.put("address", contentDTO.getAddress()) + schoolObject.put("name", contentDTO.getName()) schools.add(schoolObject) } return schools