Merge branch 'master' of http://gitlab.fangjia.com/dev0/main
This commit is contained in:
+1
-1
@@ -218,7 +218,7 @@ class CommonDatautils {
|
||||
q.put("visible", 0);
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name", "工商大楼");
|
||||
// q.put("name","东寓新村")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
|
||||
+9
-9
@@ -121,15 +121,15 @@ class AbcEvaluationRowsBatch extends BaseBatch{
|
||||
houseLoudng.fitment_cost = ret.fitment_charge
|
||||
houseLoudng.car_port = ret.carport
|
||||
houseLoudng.house_height = ret.house_hight
|
||||
houseLoudng.worry_facility" : null,
|
||||
houseLoudng.fitment_age = ret.fitment_age
|
||||
houseLoudng.lighting_ventilation" : null,
|
||||
houseLoudng.build_type = ret.build_type
|
||||
houseLoudng.house_property" : null,
|
||||
houseLoudng.noise" : null,
|
||||
houseLoudng.sight" : null,
|
||||
houseLoudng.fitment_rank" : null,
|
||||
houseLoudng.sideset" : null,
|
||||
// houseLoudng.worry_facility" : null,
|
||||
// houseLoudng.fitment_age = ret.fitment_age
|
||||
// houseLoudng.lighting_ventilation" : null,
|
||||
// houseLoudng.build_type = ret.build_type
|
||||
// houseLoudng.house_property" : null,
|
||||
// houseLoudng.noise" : null,
|
||||
// houseLoudng.sight" : null,
|
||||
// houseLoudng.fitment_rank" : null,
|
||||
// houseLoudng.sideset" : null,
|
||||
|
||||
|
||||
houseLoudng.district_no = district._id
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,117 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
class LogStat {
|
||||
|
||||
static main(args) {
|
||||
def task = new LogStat()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\建行匹配小区2.xlsx", 0, 1)
|
||||
println datas.size()
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\建行匹配小区2.xlsx",{ d ->
|
||||
def calDate = DateUtil.parseDate(d['D'], "yyyy-MM-dd")
|
||||
def weekend = DateUtils.getFirstDayOfWeek(calDate)
|
||||
def dbo = CommonDatautils.queryDistrictPrice(d['A'], d['B'], d['C'], weekend)
|
||||
if(dbo){
|
||||
d['E'] = dbo.avg_price
|
||||
}
|
||||
|
||||
def district = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
|
||||
if(district){
|
||||
def startDate = DateUtil.addMonth(calDate, -6)
|
||||
def start = startDate.replace("-","").toInteger()
|
||||
def end = DateUtil.formatByPattern(calDate,"yyyyMMdd").toInteger()
|
||||
def alias = []
|
||||
if(district.alias){
|
||||
if(!district.alias.contains(district.name)){
|
||||
district.alias << district.name
|
||||
}
|
||||
alias = district.alias
|
||||
}else{
|
||||
alias << district.name
|
||||
}
|
||||
def count = findHouses(d['A'], d['B'], d['C'], district.alias, start, end)
|
||||
d['G'] = count
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def findHouses(def city, def region, def name, def alias, def startDate, def endDate){
|
||||
|
||||
String sql = "select * from seawater-20* where cat=sell and city='${city}' and region='${region}' and district_name in (${alias.join(',')})"
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
rets << data
|
||||
println data
|
||||
})
|
||||
return rets.size()
|
||||
}
|
||||
|
||||
// def findTestDatas(def city, def value) {
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("user_id", "1007751")
|
||||
// q.put("parameter_map.city", city)
|
||||
// q.put("request_uri", "/property/evaluateForTest")
|
||||
//
|
||||
// def keys = new BasicDBObject()
|
||||
// keys.put("res_map",1)
|
||||
// def cursor = LOG.find(q)
|
||||
// def count = 0
|
||||
// println "总量:"+cursor.count()
|
||||
// def datas = []
|
||||
// for(def d : cursor){
|
||||
// if(d && d.code==200 && d.res_map && d.res_map.code==200){
|
||||
// def resMap = d.res_map.result
|
||||
// if(resMap.calcType==value){
|
||||
// def data = []
|
||||
// def rowMap = [:]
|
||||
// rowMap.city = resMap.city
|
||||
// rowMap.region = resMap.district
|
||||
// rowMap.name = resMap.name
|
||||
// rowMap.calDate = d.parameter_map.calDate
|
||||
// data << rowMap
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// return datas;
|
||||
// }
|
||||
|
||||
// for(def i=0;i<datas.size();i++){
|
||||
// def d = datas[i];
|
||||
// def calDate = DateUtil.parseDate(d['D'], "yyyy-MM-dd")
|
||||
// def weekend = DateUtils.getFirstDayOfWeek(calDate)
|
||||
// def dbo = CommonDatautils.queryDistrictPrice(d['A'], d['B'], d['C'], weekend)
|
||||
// if(dbo){
|
||||
// d['E'] = dbo.avg_price
|
||||
// }
|
||||
//
|
||||
// def district = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
|
||||
// if(district){
|
||||
// def startDate = DateUtil.addMonth(calDate, -6)
|
||||
// def start = startDate.replace("-","").toInteger()
|
||||
// def end = DateUtil.formatByPattern(calDate,"yyyyMMdd").toInteger()
|
||||
// def alias = []
|
||||
// if(district.alias){
|
||||
// if(!district.alias.contains(district.name)){
|
||||
// district.alias << district.name
|
||||
// }
|
||||
// alias = district.alias
|
||||
// }else{
|
||||
// alias << district.name
|
||||
// }
|
||||
// def count = findHouses(d['A'], d['B'], d['C'], district.alias, start, end)
|
||||
// d['G'] = count
|
||||
// }
|
||||
|
||||
|
||||
|
||||
}
|
||||
+175
@@ -0,0 +1,175 @@
|
||||
package com.fangjia.script.gj.address
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
class AddressParser {
|
||||
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
static main(args) {
|
||||
def task = new AddressParser();
|
||||
task.run()
|
||||
// task.gaoDeAddress()
|
||||
}
|
||||
|
||||
def gaoDeAddress(){
|
||||
def datas = findAllDatas()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
final AtomicInteger count = new AtomicInteger()
|
||||
for(def i=0;i<datas.size;i++){
|
||||
def data = datas[i]
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
count.addAndGet(1)
|
||||
println count.get() +" "+data.district_name
|
||||
match2(data._id, data.city, data.region, data.school_address, data.school_name);
|
||||
|
||||
} catch (Exception e) {
|
||||
println "异常"
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
latch.await()
|
||||
|
||||
}
|
||||
|
||||
def run(){
|
||||
def datas = findAllDatas()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
final AtomicInteger count = new AtomicInteger()
|
||||
for(def i=0;i<datas.size;i++){
|
||||
def data = datas[i]
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
count.addAndGet(1)
|
||||
println count.get() +" "+data.district_name
|
||||
match(data._id, data.city, data.region, data.school_address, data.school_name);
|
||||
|
||||
} catch (Exception e) {
|
||||
println "异常"
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
latch.await()
|
||||
|
||||
}
|
||||
|
||||
def match(id, city, region, address, name){
|
||||
def locationMap = null;
|
||||
// if(region && !name.contains(region)){
|
||||
// locationMap = matchLocation(city, region + name, "name")
|
||||
// }else{
|
||||
locationMap = matchLocation(city + region, name, "name")
|
||||
// }
|
||||
// if(!locationMap){
|
||||
// locationMap = matchLocation(city, address, "address")
|
||||
// }
|
||||
if(locationMap){
|
||||
DBObject data = new BasicDBObject()
|
||||
data.put("school_lat1", locationMap.glat)
|
||||
data.put("school_lng1", locationMap.glng)
|
||||
district_school_col.update(new BasicDBObject('_id', id), new BasicDBObject('$set', data))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def match2(id, city, region, address, name){
|
||||
def locationMap = null;
|
||||
|
||||
locationMap = matchLocation2(city + region, name, "name")
|
||||
|
||||
if(!locationMap){
|
||||
locationMap = matchLocation2(city, address, "address")
|
||||
}
|
||||
if(locationMap){
|
||||
DBObject data = new BasicDBObject()
|
||||
data.put("gaode_name", locationMap.name)
|
||||
data.put("gaode_address", locationMap.address)
|
||||
if(locationMap.location){
|
||||
def arrs = locationMap.location.split(",")
|
||||
data.put("school_lat", arrs[1].toDouble())
|
||||
data.put("school_lng", arrs[0].toDouble())
|
||||
}
|
||||
district_school_col.update(new BasicDBObject('_id', id), new BasicDBObject('$set', data))
|
||||
}
|
||||
}
|
||||
|
||||
def matchLocation(city, address, cat){
|
||||
|
||||
JSONObject gloJson = GaoDeApiUtils.geo(city, address)
|
||||
def map = [:]
|
||||
if (gloJson && gloJson.get("geocodes")) {
|
||||
def geocodes = gloJson.get("geocodes")
|
||||
def datas = geocodes.get(0)
|
||||
if(datas.formatted_address && cat=="name"){
|
||||
if(datas.formatted_address.contains(address)){
|
||||
def glocation = datas.location
|
||||
def los = glocation.split(",")
|
||||
map.glng = Double.parseDouble(los[0].toString())
|
||||
map.glat = Double.parseDouble(los[1].toString())
|
||||
}else{
|
||||
return null
|
||||
}
|
||||
}else{
|
||||
def glocation = geocodes.get(0).location
|
||||
def los = glocation.split(",")
|
||||
map.glng = Double.parseDouble(los[0].toString())
|
||||
map.glat = Double.parseDouble(los[1].toString())
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
def matchLocation2(city, address, cat){
|
||||
|
||||
//JSONObject gloJson = GaoDeApiUtils.geo(city, address)
|
||||
JSONObject gloJson = GaoDeApiUtils.search(address, city, '140000')
|
||||
def map = [:]
|
||||
if (gloJson && gloJson.get("pois")) {
|
||||
def geocodes = gloJson.get("pois")
|
||||
def datas = geocodes.get(0)
|
||||
return datas
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def findAllDatas(){
|
||||
def q = new BasicDBObject()
|
||||
// q.put("school_name","苏州市娄葑实验小学")
|
||||
// q.put("school_lng", new BasicDBObject(QueryOperators.EXISTS, false))
|
||||
def dbcursor = district_school_col.find(q)
|
||||
def rets =[]
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
}
|
||||
+354
@@ -0,0 +1,354 @@
|
||||
package com.fangjia.script.gj.ccb
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.FangJiaAreaMapUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.QueryOperators
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class DistrictSchoolDataExport {
|
||||
|
||||
def cityCodeMap = ["苏州":"320500","合肥":"340100", "西安":"610000"]
|
||||
|
||||
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
|
||||
static DBCollection school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "school")
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
def regionCodeMap = [:]
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictSchoolDataExport()
|
||||
// task.run()
|
||||
task.megre()
|
||||
}
|
||||
|
||||
def megre(){
|
||||
def datas1 = ExcelUtils.read("d:\\数据\\楼盘对口学校_V2.xlsx",0,0)
|
||||
def ids = []
|
||||
for(def d : datas1){
|
||||
ids << d['U']
|
||||
}
|
||||
def datas2 = ExcelUtils.read("d:\\数据\\无学区房数据(1).xlsx",0,1)
|
||||
for(def d : datas2){
|
||||
def id = d['U']
|
||||
if(!ids.contains(id)){
|
||||
datas1 << d
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\楼盘对口学校V1.xlsx", datas1, "数据")
|
||||
}
|
||||
|
||||
def run(){
|
||||
def city = "苏州"
|
||||
getRegionCodeByCity(city)
|
||||
getRegionCodeByCity("合肥")
|
||||
getRegionCodeByCity("西安")
|
||||
def districts0 = CommonDatautils.findAllDistricts("苏州", "district", ["city", "region", "name", "address", "lng", "lat", "c_date","m_date"]);
|
||||
def districts1 = CommonDatautils.findAllDistricts("合肥", "district", ["city", "region", "name", "address", "lng", "lat", "c_date","m_date"]);
|
||||
def districts2 = CommonDatautils.findAllDistricts("西安", "district", ["city", "region", "name", "address", "lng", "lat", "c_date","m_date"]);
|
||||
def districts = []
|
||||
districts.addAll(districts0)
|
||||
districts.addAll(districts1)
|
||||
districts.addAll(districts2)
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
def rets = []
|
||||
rets << ["城市","区域","小区名称","小区经度","小区纬度","小区地址","对口小学","小学地址","小学属性","小学经度","小学纬度","对口中学","中学地址","中学属性","中学经度","中学纬度","备注","创建时间","修改时间","来源网站","小区ID","数据类型"]
|
||||
def i = 0
|
||||
for(def k=0;k<districts.size();k++){
|
||||
final def dist = districts.get(k)
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
|
||||
def primarySchools = findPrimarySchool(dist.city, dist.region, dist.name)
|
||||
def midSchools = findMiddleSchool(dist.city, dist.region, dist.name)
|
||||
println totalCount.get() +" "+dist.name +" 小学:"+primarySchools.size()+" 初中:"+midSchools.size()
|
||||
if(primarySchools || midSchools){
|
||||
def info1 = null;
|
||||
def info2 = null;
|
||||
def distance1 = 0
|
||||
def distance2 = 0
|
||||
if(primarySchools && dist.lng){
|
||||
info1 = selectSchool(dist.lng/10000000000.0, dist.lat/10000000000.0, primarySchools);
|
||||
if(info1 && info1.school_lat2){
|
||||
double lat = dist.lat/10000000000.0
|
||||
double lng = dist.lng/10000000000.0
|
||||
distance1 = GeoHelper.GetDistance(lat, lng, info1.school_lat2.toDouble(), info1.school_lng2.toDouble())
|
||||
if(distance1>3000){
|
||||
info1 = null
|
||||
}
|
||||
}else{
|
||||
info1 = null
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(midSchools) {
|
||||
info2 = selectSchool(dist.lng / 10000000000.0, dist.lat / 10000000000.0, midSchools);
|
||||
if(info2 && info2.school_lng2){
|
||||
double lat = dist.lat/10000000000.0
|
||||
double lng = dist.lng/10000000000.0
|
||||
distance2 = GeoHelper.GetDistance(lat, lng, info2.school_lat2, info2.school_lng2)
|
||||
if(distance2>3000){
|
||||
info2 = null
|
||||
}
|
||||
}else{
|
||||
info2 = null
|
||||
}
|
||||
}
|
||||
if(info1|| info2){
|
||||
def row = []
|
||||
row << cityCodeMap.get(dist.city)
|
||||
row << regionCodeMap.get(dist.city +"-"+dist.region)
|
||||
row << dist.name
|
||||
row << dist.lng/10000000000.0
|
||||
row << dist.lat/10000000000.0
|
||||
row << dist.address
|
||||
if(info1){
|
||||
row << info1.school_name
|
||||
row << info1.school_address!=null?info1.school_address:''
|
||||
row << info1.attribute!=null?info1.attribute:''
|
||||
row << info1.school_lng2!=null?info1.school_lng2:''
|
||||
row << info1.school_lat2!=null?info1.school_lat2:''
|
||||
// row << distance1
|
||||
}else{
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
// row << ""
|
||||
}
|
||||
|
||||
if(info2){
|
||||
row << info2.school_name
|
||||
row << info2.school_address!=null?info2.school_address:''
|
||||
row << info2.attribute!=null?info2.attribute:''
|
||||
row << info2.school_lng2!=null?info2.school_lng2:''
|
||||
row << info2.school_lat2!=null?info2.school_lat2:''
|
||||
// row << distance2
|
||||
}else{
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
// row << ""
|
||||
}
|
||||
row << ""
|
||||
row << DateUtils.formatDateByPattern(dist.c_date, "yyyy-MM-dd HH:mm:ss")
|
||||
row << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
row << "fjw"
|
||||
row << dist._id.toString()
|
||||
row << "对口"
|
||||
|
||||
rets << row
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
ExcelUtils.write("d:\\数据\\楼盘对口学校_V2.xlsx", rets, "数据")
|
||||
|
||||
}
|
||||
|
||||
|
||||
def selectSchool(lng, lat, def schools){
|
||||
def minSchool = null
|
||||
def level0 = []//国家
|
||||
def level1 = []//省重点
|
||||
def level2 = []//市重点
|
||||
def level3 = []//区重点
|
||||
def level4 = []//普通
|
||||
def level5 = []//其它
|
||||
for(def s : schools){
|
||||
if(s.attribute){
|
||||
if(s.attribute.contains("国家重点")){
|
||||
level0 << s
|
||||
}else if(s.attribute.contains("省重点")){
|
||||
level1 << s
|
||||
}else if(s.attribute.contains("市重点")){
|
||||
level2 << s
|
||||
}else if(s.attribute.contains("区重点")){
|
||||
level3 << s
|
||||
}else if(s.attribute.contains("普通")){
|
||||
level4 << s
|
||||
}else{
|
||||
level5 << s
|
||||
}
|
||||
}else{
|
||||
level5 << s
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(level0){
|
||||
if(level0.size()==1){
|
||||
minSchool = level0.get(0)
|
||||
}else{
|
||||
minSchool = selectMinDistanceSchool(lat, lng, level0)
|
||||
}
|
||||
}else if(level1){
|
||||
if(level1.size()==1){
|
||||
minSchool = level1.get(0)
|
||||
}else{
|
||||
minSchool = selectMinDistanceSchool(lat, lng, level1)
|
||||
}
|
||||
}else if(level2){
|
||||
if(level2.size()==1){
|
||||
minSchool = level2.get(0)
|
||||
}else{
|
||||
minSchool = selectMinDistanceSchool(lat, lng, level2)
|
||||
}
|
||||
}else if(level3){
|
||||
if(level3.size()==1){
|
||||
minSchool = level3.get(0)
|
||||
}else{
|
||||
minSchool = selectMinDistanceSchool(lat, lng, level3)
|
||||
}
|
||||
}else if(level4){
|
||||
if(level4.size()==1){
|
||||
minSchool = level4.get(0)
|
||||
}else{
|
||||
minSchool = selectMinDistanceSchool(lat, lng, level4)
|
||||
}
|
||||
}else if(level5){
|
||||
if(level5.size()==1){
|
||||
minSchool = level5.get(0)
|
||||
}else{
|
||||
minSchool = selectMinDistanceSchool(lat, lng, level5)
|
||||
}
|
||||
}
|
||||
|
||||
def rets = []
|
||||
|
||||
if(minSchool){
|
||||
rets = findSchoolInfo(minSchool.city, minSchool.match_region,minSchool.school_name)
|
||||
}
|
||||
|
||||
if(rets.size()>0){
|
||||
for(def d : rets){
|
||||
if(!minSchool.school_address){
|
||||
minSchool.school_address = d.address
|
||||
}
|
||||
if(!minSchool.attribute){
|
||||
minSchool.attribute = d.attribute
|
||||
}
|
||||
}
|
||||
}else{
|
||||
if(minSchool){
|
||||
if(minSchool.school_map_lng){
|
||||
minSchool.school_lng2 = minSchool.school_map_lng
|
||||
minSchool.school_lat2 = minSchool.school_map_lat
|
||||
}else if(minSchool.school_lng1){
|
||||
minSchool.school_lng2 = minSchool.school_lng1
|
||||
minSchool.school_lat2 = minSchool.school_lat1
|
||||
}else if(minSchool.school_lng){
|
||||
minSchool.school_lng2 = minSchool.school_lng
|
||||
minSchool.school_lat2 = minSchool.school_lat
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
return minSchool;
|
||||
}
|
||||
|
||||
|
||||
def findSchoolInfo(def city, def region, def schoolName){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", schoolName)
|
||||
def dbcursor = school_col.find(q)
|
||||
def rets = []
|
||||
for(def obj : dbcursor){
|
||||
rets << obj
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
def selectMinDistanceSchool(def lat, def lng, def schools){
|
||||
def minSchool = null;
|
||||
def minDistance = Integer.MAX_VALUE
|
||||
for(def s1 : schools){
|
||||
|
||||
def lng2 = null
|
||||
def lat2 = null
|
||||
if(s1.school_map_lng){
|
||||
lng2 = s1.school_map_lng
|
||||
lat2 = s1.school_map_lat
|
||||
}else if(s1.school_lng1){
|
||||
lng2 = s1.school_lng1
|
||||
lat2 = s1.school_lat1
|
||||
}else if(s1.school_lng){
|
||||
lng2 = s1.school_lng
|
||||
lat2 = s1.school_lat
|
||||
}
|
||||
if(lat2){
|
||||
def distance = GeoHelper.GetDistance(lat, lng, lat2, lng2)
|
||||
println s1.district_name +" "+ s1.school_name+" "+distance
|
||||
if(distance<minDistance){
|
||||
minDistance = distance
|
||||
minSchool = s1
|
||||
minSchool.school_lng2 = lng2
|
||||
minSchool.school_lat2 = lat2
|
||||
}
|
||||
}
|
||||
}
|
||||
return minSchool
|
||||
}
|
||||
|
||||
def findPrimarySchool(def city, def region, def name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("match_city", city)
|
||||
q.put("match_region", region)
|
||||
q.put("match_name", name)
|
||||
q.put("category", Pattern.compile("^.*小学.*\$", Pattern.CASE_INSENSITIVE))
|
||||
def dbcursor = district_school_col.find(q).sort(new BasicDBObject("attribute", 1))
|
||||
def datas = []
|
||||
for(def db : dbcursor){
|
||||
datas << db
|
||||
}
|
||||
return datas;
|
||||
}
|
||||
|
||||
|
||||
def findMiddleSchool(def city, def region, def name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("match_city", city)
|
||||
q.put("match_region", region)
|
||||
q.put("match_name", name)
|
||||
q.put("category", new BasicDBObject(QueryOperators.NE, "小学"))
|
||||
def dbcursor = district_school_col.find(q).sort(new BasicDBObject("attribute", 1))
|
||||
def datas = []
|
||||
for(def db : dbcursor){
|
||||
datas << db
|
||||
}
|
||||
return datas;
|
||||
}
|
||||
|
||||
def getRegionCodeByCity(def city){
|
||||
def datas = FangJiaAreaMapUtils.convertToOfficials(city);
|
||||
for(def d : datas){
|
||||
regionCodeMap.put(city+"-"+d.fangjia_district, d.district_code)
|
||||
}
|
||||
}
|
||||
}
|
||||
+182
@@ -0,0 +1,182 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.fangjia.script.util.DbUtils
|
||||
|
||||
class FZloationUpdate {
|
||||
|
||||
static main(args) {
|
||||
def task = new FZloationUpdate();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region","name","address","lat","lng"]);
|
||||
def index = 0
|
||||
def locationMap = getMoreSameLocationDist(districts)
|
||||
def total = 0
|
||||
locationMap.each {k,v->
|
||||
total = updateLocation(v, total);
|
||||
println "---------------------------------------------------------------------------------------------------------------"
|
||||
}
|
||||
}
|
||||
|
||||
def updateLocation(def lists, def total){
|
||||
|
||||
|
||||
for(def dist : lists){
|
||||
|
||||
def addrDistMap = getLocation(dist, dist.address)
|
||||
def nameDistMap = getLocation(dist, dist.region + dist.name)
|
||||
|
||||
if((addrDistMap && addrDistMap.distance>1000) && (nameDistMap && nameDistMap.distance>1000)){
|
||||
println dist.region +" "+dist.name+" "+dist.address+" addrDist:"+(addrDistMap!=null?addrDistMap.distance:"")+" addrKey:"+(addrDistMap!=null?addrDistMap.keyword:"")+" nameDist:"+(nameDistMap!=null?nameDistMap.distance:"")+" nameKey:"+(nameDistMap!=null?nameDistMap.keyword:"")
|
||||
// println "[${index}]"+ dist.region+" "+dist.name+" "+dist.address+" addrDist:"+addrDistMap.distance+"(${addrDistMap.level}-${addrDistMap.location}-${addrDistMap.keyword}) nameDist:"+nameDistMap.distance+"(${nameDistMap.level}-${nameDistMap.location}-${nameDistMap.keyword})"
|
||||
|
||||
if(addrDistMap){
|
||||
def addrKey = deleteRegion(addrDistMap.keyword, dist.region).replace("|","");
|
||||
def nameKey = deleteRegion(nameDistMap.keyword, dist.region);
|
||||
|
||||
def baidu = null
|
||||
def lat = null
|
||||
def lng = null
|
||||
def upldateFlag = ""
|
||||
if(addrDistMap && addrDistMap.region.contains(dist.region) && (addrKey.contains("路") || addrKey.contains("大道")) && addrKey.contains("号") && dist.addr == addrKey){
|
||||
lat = addrDistMap.lat
|
||||
lng = addrDistMap.lng
|
||||
baidu = BaiDuApiUtils.geoconv(lng, lat, 3)
|
||||
upldateFlag = "Address"
|
||||
}else if(nameDistMap && nameDistMap.region.contains(dist.region) && nameKey == dist.name){
|
||||
lng = nameDistMap.lng
|
||||
lat = nameDistMap.lat
|
||||
baidu = BaiDuApiUtils.geoconv(lng, lat, 3)
|
||||
upldateFlag = "Name"
|
||||
}
|
||||
if(baidu){
|
||||
total++
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("m_date", new Date())
|
||||
Long baiduLat = baidu.y*10000000000L
|
||||
Long baiduLng = baidu.x*10000000000L
|
||||
udata.put("lat2", baiduLat)
|
||||
udata.put("lng2", baiduLng)
|
||||
Long lat0 = lat*10000000000L
|
||||
Long lng0 = lng*10000000000L
|
||||
udata.put("lat", lat0)
|
||||
udata.put("lng", lng0 )
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
println "[修改-${total}] "+ dist.region+" "+dist.name+" "+dist.address+" "+upldateFlag
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
def getMoreSameLocationDist(def districts){
|
||||
def locationMap = [:]
|
||||
for(def i=0;i<districts.size();i++){
|
||||
def dist = districts[i]
|
||||
def lat = dist.lat/10000000000.0;
|
||||
def lng = dist.lng/10000000000.0;
|
||||
def locationKey = lng + "-" + lat
|
||||
def list = locationMap.get(locationKey)
|
||||
if(!list){
|
||||
list = []
|
||||
locationMap.put(locationKey, list);
|
||||
}
|
||||
list << dist
|
||||
}
|
||||
def resMap = [:]
|
||||
locationMap.each{k,v->
|
||||
if(v.size()>=2){
|
||||
|
||||
v.sort{ a,b->
|
||||
return a.name.size().compareTo(b.name.size())
|
||||
}
|
||||
resMap.put(k,v)
|
||||
}
|
||||
}
|
||||
|
||||
def lastRetMaps = [:]
|
||||
resMap.each{k,v->
|
||||
def swname = null
|
||||
def count = 0
|
||||
for(def i=0;i<v.size();i++){
|
||||
def d = v[i]
|
||||
if(i==0){
|
||||
if(d.name.contains("期")||d.name.contains("区")){
|
||||
swname = d.name.replace("一期","").replace("二期","").replace("三期","").replace("四期","").replace("五期","").replace("六期","").replace("一区","").replace("二区","").replace("三区","").replace("四区","").replace("五区","").replace("A区","").replace("B区","").replace("C区","").replace("D区","")
|
||||
}else{
|
||||
swname = d.name
|
||||
}
|
||||
}else{
|
||||
if(d.name.contains(swname)||swname.contains(d.name)){
|
||||
count++
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
if(count+1!=v.size()){
|
||||
lastRetMaps.put(k,v)
|
||||
}
|
||||
}
|
||||
return lastRetMaps;
|
||||
}
|
||||
|
||||
|
||||
def deleteRegion(def s, def region){
|
||||
def key = ""
|
||||
if(s.indexOf(region)>-1){
|
||||
if(s.indexOf(region+"区")>-1){
|
||||
key = s.substring(s.indexOf(region+"区")+region.size()+1, s.size());
|
||||
}else if(s.indexOf(region+"县")>-1){
|
||||
key = s.substring(s.indexOf(region+"县")+region.size()+1, s.size());
|
||||
}else if(s.indexOf(region+"市")>-1){
|
||||
key = s.substring(s.indexOf(region+"市")+region.size()+1, s.size());
|
||||
}
|
||||
}else{
|
||||
key = s;
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
|
||||
|
||||
def getLocation(dist, addr){
|
||||
|
||||
if(addr){
|
||||
def rMap = [:]
|
||||
def reps = GaoDeApiUtils.geo(dist.city, addr);
|
||||
if(reps && reps.count.toInteger()>=1){
|
||||
def map = reps.geocodes.get(0)
|
||||
def location = map.location
|
||||
def level = map.level
|
||||
if(location){
|
||||
def locMap = location.split(",")
|
||||
def lat = dist.lat/10000000000.0;
|
||||
def lng = dist.lng/10000000000.0;
|
||||
def distance = GeoHelper.GetDistance(locMap[1].toDouble(), locMap[0].toDouble(), lat, lng)
|
||||
rMap.distance = distance
|
||||
rMap.level = level
|
||||
rMap.lat = locMap[1].toDouble()
|
||||
rMap.lng = locMap[0].toDouble()
|
||||
rMap.keyword = map.formatted_address
|
||||
rMap.location = location
|
||||
rMap.region = map.district
|
||||
return rMap;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
class FzLocationBatch {
|
||||
}
|
||||
+20
-1
@@ -30,7 +30,9 @@ class JiangShuBlankDataStat {
|
||||
// task.matchHouse();
|
||||
|
||||
// task.villa()
|
||||
task.addressParse()
|
||||
// task.addressParse()
|
||||
|
||||
task.deleteProcess()
|
||||
}
|
||||
|
||||
def repeat(){
|
||||
@@ -288,5 +290,22 @@ class JiangShuBlankDataStat {
|
||||
return floor;
|
||||
}
|
||||
|
||||
def deleteProcess(){
|
||||
|
||||
def datas = ExcelUtils.read("D:\\数据\\江苏农行\\江苏2千.xlsx",0,1)
|
||||
def addrList = []
|
||||
for(def d : datas){
|
||||
def addr = d['G']
|
||||
addrList << addr
|
||||
}
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\200平以下.xlsx", { row ->
|
||||
def addr = row['C']
|
||||
if(addrList.contains(addr)){
|
||||
row['R'] = "已调查"
|
||||
}
|
||||
|
||||
},0);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
println "wwww"
|
||||
+2
-2
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
|
||||
def run(){
|
||||
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"]
|
||||
for(def source :sources){
|
||||
def count1 = findTotalCount(source, 202101);
|
||||
def count2 = findTotalCount(source, 202102);
|
||||
def count1 = findTotalCount(source, 202102);
|
||||
def count2 = findTotalCount(source, 202103);
|
||||
println source +" "+count1 +" "+count2
|
||||
}
|
||||
}
|
||||
|
||||
+213
@@ -0,0 +1,213 @@
|
||||
package com.fangjia.script.thirdPriceCrawl.other
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
import net.sf.json.groovy.JsonSlurper
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
|
||||
class AddFangSchoolRegion {
|
||||
|
||||
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
|
||||
static DBCollection third_price_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "third_party_price")
|
||||
static DBCollection school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "school2")
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
static main(args) {
|
||||
def task = new AddFangSchoolRegion();
|
||||
//task.process()
|
||||
//task.match()
|
||||
task.addMapLocation()
|
||||
}
|
||||
|
||||
def addMapLocation(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = district_school_col.find(q)
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
def data = findSchool(d.city, d.region, d.school_name)
|
||||
if(data && data.location){
|
||||
def location = data.location
|
||||
if(location){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("school_map_lng", location[0])
|
||||
udata.put("school_map_lat", location[1])
|
||||
district_school_col.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', udata))
|
||||
}
|
||||
|
||||
}
|
||||
index ++
|
||||
println index
|
||||
}
|
||||
}
|
||||
|
||||
def match(){
|
||||
def datas = findAllDatas()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
for(def i=0;i<datas.size;i++){
|
||||
def data = datas[i]
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
// if(!data.match_name){
|
||||
println i +" "+data.district_name
|
||||
matching(data);
|
||||
// }
|
||||
|
||||
} catch (Exception e) {
|
||||
println "Òì³£"
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
|
||||
|
||||
def matching(def it){
|
||||
def rMap = matchDistrict(it.city, it.region, it.district_name)
|
||||
if(!rMap){
|
||||
rMap = matchDistrict(it.city, it.region, it.address)
|
||||
}
|
||||
|
||||
if(rMap){
|
||||
DBObject data = new BasicDBObject()
|
||||
data.put("match_city", rMap.city)
|
||||
data.put("match_region", rMap.region)
|
||||
data.put("match_name", rMap.name)
|
||||
data.put("match_address", rMap.address)
|
||||
data.put("match_lat", rMap.lat)
|
||||
data.put("match_lng", rMap.lng)
|
||||
district_school_col.update(new BasicDBObject('_id', it.get("_id")), new BasicDBObject('$set', data))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def matchDistrict(def city, def region, def address){
|
||||
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&preciseLevel=strict&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district"
|
||||
String content = new HttpRequest().get(url, "utf-8");
|
||||
def rMap = [:]
|
||||
if(content){
|
||||
def doc = new JsonSlurper().parseText(content)
|
||||
if(doc && doc.code ==200 && doc.result){
|
||||
def dto = doc.result
|
||||
if(dto && dto.searchAddress){
|
||||
def ret = dto.matchResult
|
||||
if(ret.match){
|
||||
rMap.city = dto.city
|
||||
rMap.region = dto.district
|
||||
rMap.name = dto.name
|
||||
rMap.lat = dto.lat
|
||||
rMap.lng = dto.lng
|
||||
rMap.id = dto.id
|
||||
rMap.address = dto.address
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return rMap;
|
||||
}
|
||||
|
||||
|
||||
def process(){
|
||||
def datas = findNoRegionData();
|
||||
findSimialData(datas);
|
||||
}
|
||||
|
||||
def findAllDatas(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("match_name", new BasicDBObject(QueryOperators.EXISTS, false))
|
||||
def dbcursor = district_school_col.find(q)
|
||||
def rets =[]
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findNoRegionData(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("region", null)
|
||||
def dbcursor = district_school_col.find(q)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findSimialData(datas){
|
||||
for(def d : datas){
|
||||
def obj = findThirdPriceData(d.city, d.url+"/")
|
||||
if(!obj){
|
||||
obj = findThirdPriceByName(d.city, d.district_name)
|
||||
}
|
||||
if(obj){
|
||||
d.region = obj.region
|
||||
if(!d.district_addrss){
|
||||
d.district_address = obj.address
|
||||
}
|
||||
updateDistrictSchool(d);
|
||||
|
||||
}else{
|
||||
println d.school_name+"" + d.district_name+"û²éµ½"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def findThirdPriceData(def city, def url){
|
||||
def q = new BasicDBObject()
|
||||
q.put("source","ËÑ·¿")
|
||||
q.put("city", city)
|
||||
q.put("district_id", url)
|
||||
def dbcursor = third_price_col.find(q).sort(new BasicDBObject("s_date", -1)).limit(1)
|
||||
def obj = null;
|
||||
for(def d : dbcursor){
|
||||
obj = d
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
def findThirdPriceByName(def city, def name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("source","ËÑ·¿")
|
||||
q.put("city", city)
|
||||
q.put("name", name)
|
||||
def dbcursor = third_price_col.find(q).sort(new BasicDBObject("s_date", -1)).limit(1)
|
||||
def obj = null;
|
||||
for(def d : dbcursor){
|
||||
obj = d
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
def updateDistrictSchool(d){
|
||||
|
||||
DBObject data = new BasicDBObject()
|
||||
data.put("region", d.region)
|
||||
data.put("district_address", d.district_address)
|
||||
district_school_col.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', data))
|
||||
}
|
||||
|
||||
def findSchool(city, region, school_name){
|
||||
DBObject data = new BasicDBObject()
|
||||
data.put("city", city)
|
||||
data.put("region", region)
|
||||
data.put("name", school_name)
|
||||
return school_col.findOne(data)
|
||||
}
|
||||
}
|
||||
+344
@@ -0,0 +1,344 @@
|
||||
package com.fangjia.script.thirdPriceCrawl.other
|
||||
|
||||
import com.alibaba.fastjson.annotation.JSONField
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.httpclient.UserAgents
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import com.fangjia.script.util.HttpClient
|
||||
import com.fangjia.script.util.JacksonUtil
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.httpclient.NameValuePair
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.http.HttpEntity
|
||||
import org.apache.http.client.entity.UrlEncodedFormEntity
|
||||
import org.apache.http.client.methods.CloseableHttpResponse
|
||||
import org.apache.http.client.methods.HttpPost
|
||||
import org.apache.http.impl.client.CloseableHttpClient
|
||||
import org.apache.http.impl.client.HttpClients
|
||||
import org.apache.http.message.BasicNameValuePair
|
||||
import org.apache.http.util.EntityUtils
|
||||
|
||||
import java.nio.charset.Charset
|
||||
import java.util.zip.GZIPInputStream
|
||||
|
||||
class GushuwangCrawl {
|
||||
|
||||
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
|
||||
static DBCollection school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "school")
|
||||
static main(args) {
|
||||
def task = new GushuwangCrawl();
|
||||
task.process()
|
||||
}
|
||||
|
||||
|
||||
void process(Object args) {
|
||||
|
||||
// chaxuequCrawl()
|
||||
suzhouCrawl();
|
||||
|
||||
}
|
||||
|
||||
def suzhouCrawl(){
|
||||
def page = 1
|
||||
def stop = false
|
||||
def list = []
|
||||
while(!stop){
|
||||
def s = schoolList(page);
|
||||
if(s){
|
||||
JSONObject jsonObject = JSONObject.fromObject(s);
|
||||
if(jsonObject && jsonObject.ok){
|
||||
def datas = jsonObject.extra.list
|
||||
if(datas){
|
||||
list.addAll(datas)
|
||||
page++
|
||||
println page
|
||||
}else{
|
||||
stop = true
|
||||
}
|
||||
}else{
|
||||
stop = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(list){
|
||||
def schools = []
|
||||
for(def d : list){
|
||||
def schoolData = new BasicDBObject()
|
||||
schoolData.province = "江苏"
|
||||
schoolData.city = "苏州"
|
||||
schoolData.region = RegionData.parseRegionFormAddress(schoolData.city , d.adr)
|
||||
/**类型: 幼儿园,小学,初中*/
|
||||
schoolData.category = converCategory(d.schoolname)
|
||||
schoolData.name = d.schoolname
|
||||
schoolData.address = d.adr
|
||||
/**来源网站*/
|
||||
schoolData.source = "苏州学区查询"
|
||||
/**房源详情url*/
|
||||
schools << schoolData
|
||||
}
|
||||
|
||||
saveChaxuequSchool(schools)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
def chaxuequCrawl(){
|
||||
|
||||
def page = 1
|
||||
def stop = false
|
||||
|
||||
def schools = []
|
||||
|
||||
while(!stop){
|
||||
def s = chaxuequInfoCrawl("https://51yongka.com/fapi/school/list", "schoolName=&city=%E8%8B%8F%E5%B7%9E&area=&schoolNature=&schoolType=&page="+page+"&size=10")
|
||||
println "内容:"+s
|
||||
if(s){
|
||||
JSONObject jsonObject = JSONObject.fromObject(s);
|
||||
if(jsonObject.code == 200){
|
||||
def data = jsonObject.data
|
||||
def totalPage = data.pages
|
||||
if(data.list){
|
||||
page++
|
||||
if(page+1>totalPage){
|
||||
stop = true;
|
||||
}
|
||||
schools.addAll(data.list)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
print(schools.size());
|
||||
|
||||
def districtSchools = []
|
||||
def schoolInfos = []
|
||||
if(schools.size()>0){
|
||||
|
||||
def url = "https://51yongka.com/fapi/school/details"
|
||||
for(def d : schools){
|
||||
def params = "schoolId="+d.id+"&access_token=oRqu15Mq_KxHfK0y0ZWhwktVnfUM"
|
||||
def s = chaxuequInfoCrawl(url, params)
|
||||
JSONObject jsonObject = JSONObject.fromObject(s);
|
||||
if(jsonObject && jsonObject.code==200){
|
||||
|
||||
if(jsonObject.data && jsonObject.data.nearbyPropertyList){
|
||||
def districts = jsonObject.data.nearbyPropertyList
|
||||
for(def dist : districts){
|
||||
def dataMap = new BasicDBObject()
|
||||
dataMap.city = jsonObject.data.city
|
||||
dataMap.region = RegionData.formatRegion(dataMap.city , jsonObject.data.area)
|
||||
/**类型: 幼儿园,小学,初中*/
|
||||
dataMap.category = converCategory(jsonObject.data.schoolName)
|
||||
dataMap.district_name = dist.name.trim()
|
||||
dataMap.district_address = dist.address
|
||||
dataMap.school_name = jsonObject.data.schoolName.trim()
|
||||
dataMap.school_address = jsonObject.data.address
|
||||
dataMap.source = "查学区房"
|
||||
dataMap.url = url + "?"+params
|
||||
dataMap.telephone = jsonObject.data.officialTelephone;
|
||||
districtSchools << dataMap
|
||||
}
|
||||
}else{
|
||||
|
||||
def schoolData = new BasicDBObject()
|
||||
schoolData.province = "江苏"
|
||||
schoolData.city = jsonObject.data.city
|
||||
schoolData.region = RegionData.formatRegion(schoolData.city , jsonObject.data.area)
|
||||
/**类型: 幼儿园,小学,初中*/
|
||||
schoolData.category = converCategory(jsonObject.data.schoolName)
|
||||
schoolData.name = jsonObject.data.schoolName.trim()
|
||||
schoolData.address = jsonObject.data.address
|
||||
/**来源网站*/
|
||||
schoolData.source = "查学区房"
|
||||
/**房源详情url*/
|
||||
schoolData.telephone = jsonObject.data.officialTelephone;
|
||||
schoolInfos << schoolData
|
||||
println jsonObject.data.schoolName +" 无周边楼盘"
|
||||
}
|
||||
|
||||
}
|
||||
println s;
|
||||
}
|
||||
}
|
||||
|
||||
println "小区:"+districtSchools.size()
|
||||
// saveChaxuequDistrictSchool(districtSchools)
|
||||
saveChaxuequSchool(schoolInfos)
|
||||
|
||||
}
|
||||
|
||||
def saveChaxuequSchool(schoolInfos){
|
||||
for(def ds : schoolInfos){
|
||||
def obj = findSchoolData(ds.city, ds.school_name);
|
||||
if(!obj){
|
||||
school_col.save(ds)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def saveChaxuequDistrictSchool(def districtSchools){
|
||||
for(def ds : districtSchools){
|
||||
def obj = findDistrictSchoolData(ds.city, ds.school_name);
|
||||
if(!obj){
|
||||
district_school_col.save(ds)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def findSchoolData(city, school_name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("school_name", school_name)
|
||||
return school_col.findOne(q)
|
||||
}
|
||||
|
||||
|
||||
def findDistrictSchoolData(city, school_name, district_name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("school_name", school_name)
|
||||
q.put("district_name", district_name)
|
||||
return district_school_col.findOne(q)
|
||||
}
|
||||
|
||||
def converCategory(def schoolName){
|
||||
|
||||
if(schoolName){
|
||||
if(schoolName.contains("初中部")||schoolName.contains("中学")){
|
||||
return "初中"
|
||||
}else if(schoolName.contains("小学")){
|
||||
return "小学"
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def static chaxuequInfoCrawl(url, bodyquery){
|
||||
def responseMap = [:]
|
||||
try {
|
||||
//headers信息
|
||||
Map<String, String> headers = new LinkedHashMap<String, String>()
|
||||
headers.put("Host","51yongka.com")
|
||||
headers.put("Connection","keep-alive")
|
||||
headers.put("Content-Length","75")
|
||||
headers.put("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat")
|
||||
headers.put("content-type","application/x-www-form-urlencoded")
|
||||
headers.put("Referer","https://servicewechat.com/wxc2461b2529576950/8/page-frame.html")
|
||||
headers.put("Accept-Encoding","gzip, deflate, br")
|
||||
String s = doPost(url, bodyquery, headers)
|
||||
return s
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
def static schoolList(page) {
|
||||
def responseMap = [:]
|
||||
String s = null
|
||||
try {
|
||||
//headers信息
|
||||
Map<String, String> headers = new LinkedHashMap<String, String>()
|
||||
headers.put("Host","m.gusuwang.com")
|
||||
headers.put("Connection","keep-alive")
|
||||
headers.put("Content-Length","40")
|
||||
headers.put("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat")
|
||||
headers.put("content-type","application/x-www-form-urlencoded")
|
||||
headers.put("Referer","https://servicewechat.com/wx9188a941e5ef95c5/10/page-frame.html")
|
||||
// headers.put("Accept-Encoding","gzip, deflate, br")
|
||||
|
||||
//params信息
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("secCode","198a53da6baa3aa6a5b99959a006be37");
|
||||
// //post请求
|
||||
def url = "https://m.gusuwang.com/xuequ/xqapi.php?act=searchschool"
|
||||
s = doPost(url, "secCode=198a53da6baa3aa6a5b99959a006be37&keys=&grade=&lb=&lx=&area=&currpage="+page,headers)
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
static String doPost(String strUrl, String content, Map headers) {
|
||||
String result = "";
|
||||
OutputStream outputStream = null;
|
||||
InputStream inputStream = null;
|
||||
StringBuilder sb = new StringBuilder();
|
||||
try {
|
||||
URL url = new URL(strUrl);
|
||||
//通过调用url.openConnection()来获得一个新的URLConnection对象,并且将其结果强制转换为HttpURLConnection.
|
||||
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
|
||||
urlConnection.setRequestMethod("POST");
|
||||
//设置连接的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
|
||||
urlConnection.setConnectTimeout(30000);
|
||||
//设置读取的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
|
||||
urlConnection.setReadTimeout(30000);
|
||||
//将url连接用于输出,这样才能使用getOutputStream()。getOutputStream()返回的输出流用于传输数据
|
||||
urlConnection.setDoOutput(true);
|
||||
urlConnection.setDoInput(true);
|
||||
//设置通用请求属性为默认浏览器编码类型
|
||||
// urlConnection.setRequestProperty("content-type", "application/x-www-form-urlencoded");
|
||||
|
||||
|
||||
if (headers != null && headers.keySet().size() > 0) {
|
||||
for (String head : headers.keySet()) {
|
||||
urlConnection.setRequestProperty(head, headers.get(head));
|
||||
}
|
||||
}
|
||||
//getOutputStream()返回的输出流,用于写入参数数据。
|
||||
outputStream = urlConnection.getOutputStream();
|
||||
outputStream.write(content.getBytes());
|
||||
outputStream.flush();
|
||||
outputStream.close();
|
||||
//此时将调用接口方法。getInputStream()返回的输入流可以读取返回的数据。
|
||||
inputStream = urlConnection.getInputStream();
|
||||
|
||||
|
||||
// byte[] data = new byte[1024];
|
||||
// //inputStream每次就会将读取1024个byte到data中,当inputSteam中没有数据时,inputStream.read(data)值为-1
|
||||
// while (inputStream.read(data) != -1) {
|
||||
// String s = new String(data, Charset.forName("utf-8"));
|
||||
// sb.append(s);
|
||||
// }
|
||||
|
||||
InputStreamReader reader = new InputStreamReader(inputStream, "UTF-8");
|
||||
char[] buff = new char[1024];
|
||||
int length = 0;
|
||||
while ((length = reader.read(buff)) != -1) {
|
||||
String s = new String(buff, 0, length);
|
||||
sb.append(s);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}finally{
|
||||
if(outputStream!=null){
|
||||
outputStream.close();
|
||||
}
|
||||
if(inputStream!=null){
|
||||
inputStream.close();
|
||||
}
|
||||
}
|
||||
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
@@ -52,7 +52,7 @@ class ElasticUtils {
|
||||
return dataSource;
|
||||
}
|
||||
|
||||
static executeSqlQuery(String sql, Closure<Map> closure, String host = "192.168.0.216"){
|
||||
static executeSqlQuery(String sql, Closure<Map> closure, String host = "192.168.0.206"){
|
||||
EsDataSource dataSource = getEsDataSource(host)
|
||||
Connection connection = dataSource.getConnection()
|
||||
Statement statement = connection.createStatement()
|
||||
|
||||
Reference in New Issue
Block a user