This commit is contained in:
2021-04-16 19:22:29 +08:00
parent 61ab9a8ada
commit 97fa0c9276
9 changed files with 2121 additions and 61 deletions
@@ -218,7 +218,7 @@ class CommonDatautils {
q.put("visible", 0);
q.put("cat", cat);
q.put("city", city);
// q.put("name", "工商大楼");
// q.put("name","东寓新村")
DBObject keys = new BasicDBObject();
if(fields){
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,117 @@
package com.fangjia.script.gj
import com.fangjia.core.util.DateUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
class LogStat {
static main(args) {
def task = new LogStat()
task.run()
}
def run(){
def datas = ExcelUtils.read("D:\\数据\\建行\\建行匹配小区2.xlsx", 0, 1)
println datas.size()
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\建行匹配小区2.xlsx",{ d ->
def calDate = DateUtil.parseDate(d['D'], "yyyy-MM-dd")
def weekend = DateUtils.getFirstDayOfWeek(calDate)
def dbo = CommonDatautils.queryDistrictPrice(d['A'], d['B'], d['C'], weekend)
if(dbo){
d['E'] = dbo.avg_price
}
def district = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
if(district){
def startDate = DateUtil.addMonth(calDate, -6)
def start = startDate.replace("-","").toInteger()
def end = DateUtil.formatByPattern(calDate,"yyyyMMdd").toInteger()
def alias = []
if(district.alias){
if(!district.alias.contains(district.name)){
district.alias << district.name
}
alias = district.alias
}else{
alias << district.name
}
def count = findHouses(d['A'], d['B'], d['C'], district.alias, start, end)
d['G'] = count
}
},0)
}
def findHouses(def city, def region, def name, def alias, def startDate, def endDate){
String sql = "select * from seawater-20* where cat=sell and city='${city}' and region='${region}' and district_name in (${alias.join(',')})"
def rets = []
ElasticUtils.executeSqlQuery(sql,{data->
rets << data
println data
})
return rets.size()
}
// def findTestDatas(def city, def value) {
// def q = new BasicDBObject()
// q.put("user_id", "1007751")
// q.put("parameter_map.city", city)
// q.put("request_uri", "/property/evaluateForTest")
//
// def keys = new BasicDBObject()
// keys.put("res_map",1)
// def cursor = LOG.find(q)
// def count = 0
// println "总量:"+cursor.count()
// def datas = []
// for(def d : cursor){
// if(d && d.code==200 && d.res_map && d.res_map.code==200){
// def resMap = d.res_map.result
// if(resMap.calcType==value){
// def data = []
// def rowMap = [:]
// rowMap.city = resMap.city
// rowMap.region = resMap.district
// rowMap.name = resMap.name
// rowMap.calDate = d.parameter_map.calDate
// data << rowMap
// }
// }
// }
// return datas;
// }
// for(def i=0;i<datas.size();i++){
// def d = datas[i];
// def calDate = DateUtil.parseDate(d['D'], "yyyy-MM-dd")
// def weekend = DateUtils.getFirstDayOfWeek(calDate)
// def dbo = CommonDatautils.queryDistrictPrice(d['A'], d['B'], d['C'], weekend)
// if(dbo){
// d['E'] = dbo.avg_price
// }
//
// def district = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
// if(district){
// def startDate = DateUtil.addMonth(calDate, -6)
// def start = startDate.replace("-","").toInteger()
// def end = DateUtil.formatByPattern(calDate,"yyyyMMdd").toInteger()
// def alias = []
// if(district.alias){
// if(!district.alias.contains(district.name)){
// district.alias << district.name
// }
// alias = district.alias
// }else{
// alias << district.name
// }
// def count = findHouses(d['A'], d['B'], d['C'], district.alias, start, end)
// d['G'] = count
// }
}
@@ -0,0 +1,175 @@
package com.fangjia.script.gj.address
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import net.sf.json.JSONObject
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
class AddressParser {
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
def task = new AddressParser();
task.run()
// task.gaoDeAddress()
}
def gaoDeAddress(){
def datas = findAllDatas()
final CountDownLatch latch = new CountDownLatch(datas.size())
final AtomicInteger count = new AtomicInteger()
for(def i=0;i<datas.size;i++){
def data = datas[i]
fixedThreadPool.execute(
new Runnable() {
void run() {
try {
count.addAndGet(1)
println count.get() +" "+data.district_name
match2(data._id, data.city, data.region, data.school_address, data.school_name);
} catch (Exception e) {
println "异常"
e.printStackTrace();
} finally {
latch.countDown()
}
}
}
);
}
latch.await()
}
def run(){
def datas = findAllDatas()
final CountDownLatch latch = new CountDownLatch(datas.size())
final AtomicInteger count = new AtomicInteger()
for(def i=0;i<datas.size;i++){
def data = datas[i]
fixedThreadPool.execute(
new Runnable() {
void run() {
try {
count.addAndGet(1)
println count.get() +" "+data.district_name
match(data._id, data.city, data.region, data.school_address, data.school_name);
} catch (Exception e) {
println "异常"
e.printStackTrace();
} finally {
latch.countDown()
}
}
}
);
}
latch.await()
}
def match(id, city, region, address, name){
def locationMap = null;
// if(region && !name.contains(region)){
// locationMap = matchLocation(city, region + name, "name")
// }else{
locationMap = matchLocation(city + region, name, "name")
// }
// if(!locationMap){
// locationMap = matchLocation(city, address, "address")
// }
if(locationMap){
DBObject data = new BasicDBObject()
data.put("school_lat1", locationMap.glat)
data.put("school_lng1", locationMap.glng)
district_school_col.update(new BasicDBObject('_id', id), new BasicDBObject('$set', data))
}
}
def match2(id, city, region, address, name){
def locationMap = null;
locationMap = matchLocation2(city + region, name, "name")
if(!locationMap){
locationMap = matchLocation2(city, address, "address")
}
if(locationMap){
DBObject data = new BasicDBObject()
data.put("gaode_name", locationMap.name)
data.put("gaode_address", locationMap.address)
if(locationMap.location){
def arrs = locationMap.location.split(",")
data.put("school_lat", arrs[1].toDouble())
data.put("school_lng", arrs[0].toDouble())
}
district_school_col.update(new BasicDBObject('_id', id), new BasicDBObject('$set', data))
}
}
def matchLocation(city, address, cat){
JSONObject gloJson = GaoDeApiUtils.geo(city, address)
def map = [:]
if (gloJson && gloJson.get("geocodes")) {
def geocodes = gloJson.get("geocodes")
def datas = geocodes.get(0)
if(datas.formatted_address && cat=="name"){
if(datas.formatted_address.contains(address)){
def glocation = datas.location
def los = glocation.split(",")
map.glng = Double.parseDouble(los[0].toString())
map.glat = Double.parseDouble(los[1].toString())
}else{
return null
}
}else{
def glocation = geocodes.get(0).location
def los = glocation.split(",")
map.glng = Double.parseDouble(los[0].toString())
map.glat = Double.parseDouble(los[1].toString())
}
}
return map
}
def matchLocation2(city, address, cat){
//JSONObject gloJson = GaoDeApiUtils.geo(city, address)
JSONObject gloJson = GaoDeApiUtils.search(address, city, '140000')
def map = [:]
if (gloJson && gloJson.get("pois")) {
def geocodes = gloJson.get("pois")
def datas = geocodes.get(0)
return datas
}
return null
}
def findAllDatas(){
def q = new BasicDBObject()
// q.put("school_name","苏州市娄葑实验小学")
// q.put("school_lng", new BasicDBObject(QueryOperators.EXISTS, false))
def dbcursor = district_school_col.find(q)
def rets =[]
for(def d : dbcursor){
rets << d
}
return rets;
}
}
@@ -0,0 +1,354 @@
package com.fangjia.script.gj.ccb
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.QueryOperators
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
class DistrictSchoolDataExport {
def cityCodeMap = ["苏州":"320500","合肥":"340100", "西安":"610000"]
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
static DBCollection school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "school")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
def regionCodeMap = [:]
static main(args) {
def task = new DistrictSchoolDataExport()
// task.run()
task.megre()
}
def megre(){
def datas1 = ExcelUtils.read("d:\\数据\\楼盘对口学校_V2.xlsx",0,0)
def ids = []
for(def d : datas1){
ids << d['U']
}
def datas2 = ExcelUtils.read("d:\\数据\\无学区房数据(1).xlsx",0,1)
for(def d : datas2){
def id = d['U']
if(!ids.contains(id)){
datas1 << d
}
}
ExcelUtils.write("d:\\数据\\楼盘对口学校V1.xlsx", datas1, "数据")
}
def run(){
def city = "苏州"
getRegionCodeByCity(city)
getRegionCodeByCity("合肥")
getRegionCodeByCity("西安")
def districts0 = CommonDatautils.findAllDistricts("苏州", "district", ["city", "region", "name", "address", "lng", "lat", "c_date","m_date"]);
def districts1 = CommonDatautils.findAllDistricts("合肥", "district", ["city", "region", "name", "address", "lng", "lat", "c_date","m_date"]);
def districts2 = CommonDatautils.findAllDistricts("西安", "district", ["city", "region", "name", "address", "lng", "lat", "c_date","m_date"]);
def districts = []
districts.addAll(districts0)
districts.addAll(districts1)
districts.addAll(districts2)
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
def rets = []
rets << ["城市","区域","小区名称","小区经度","小区纬度","小区地址","对口小学","小学地址","小学属性","小学经度","小学纬度","对口中学","中学地址","中学属性","中学经度","中学纬度","备注","创建时间","修改时间","来源网站","小区ID","数据类型"]
def i = 0
for(def k=0;k<districts.size();k++){
final def dist = districts.get(k)
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def primarySchools = findPrimarySchool(dist.city, dist.region, dist.name)
def midSchools = findMiddleSchool(dist.city, dist.region, dist.name)
println totalCount.get() +" "+dist.name +" 小学:"+primarySchools.size()+" 初中:"+midSchools.size()
if(primarySchools || midSchools){
def info1 = null;
def info2 = null;
def distance1 = 0
def distance2 = 0
if(primarySchools && dist.lng){
info1 = selectSchool(dist.lng/10000000000.0, dist.lat/10000000000.0, primarySchools);
if(info1 && info1.school_lat2){
double lat = dist.lat/10000000000.0
double lng = dist.lng/10000000000.0
distance1 = GeoHelper.GetDistance(lat, lng, info1.school_lat2.toDouble(), info1.school_lng2.toDouble())
if(distance1>3000){
info1 = null
}
}else{
info1 = null
}
}
if(midSchools) {
info2 = selectSchool(dist.lng / 10000000000.0, dist.lat / 10000000000.0, midSchools);
if(info2 && info2.school_lng2){
double lat = dist.lat/10000000000.0
double lng = dist.lng/10000000000.0
distance2 = GeoHelper.GetDistance(lat, lng, info2.school_lat2, info2.school_lng2)
if(distance2>3000){
info2 = null
}
}else{
info2 = null
}
}
if(info1|| info2){
def row = []
row << cityCodeMap.get(dist.city)
row << regionCodeMap.get(dist.city +"-"+dist.region)
row << dist.name
row << dist.lng/10000000000.0
row << dist.lat/10000000000.0
row << dist.address
if(info1){
row << info1.school_name
row << info1.school_address!=null?info1.school_address:''
row << info1.attribute!=null?info1.attribute:''
row << info1.school_lng2!=null?info1.school_lng2:''
row << info1.school_lat2!=null?info1.school_lat2:''
// row << distance1
}else{
row << ""
row << ""
row << ""
row << ""
row << ""
// row << ""
}
if(info2){
row << info2.school_name
row << info2.school_address!=null?info2.school_address:''
row << info2.attribute!=null?info2.attribute:''
row << info2.school_lng2!=null?info2.school_lng2:''
row << info2.school_lat2!=null?info2.school_lat2:''
// row << distance2
}else{
row << ""
row << ""
row << ""
row << ""
row << ""
// row << ""
}
row << ""
row << DateUtils.formatDateByPattern(dist.c_date, "yyyy-MM-dd HH:mm:ss")
row << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss")
row << "fjw"
row << dist._id.toString()
row << "对口"
rets << row
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
ExcelUtils.write("d:\\数据\\楼盘对口学校_V2.xlsx", rets, "数据")
}
def selectSchool(lng, lat, def schools){
def minSchool = null
def level0 = []//国家
def level1 = []//省重点
def level2 = []//市重点
def level3 = []//区重点
def level4 = []//普通
def level5 = []//其它
for(def s : schools){
if(s.attribute){
if(s.attribute.contains("国家重点")){
level0 << s
}else if(s.attribute.contains("省重点")){
level1 << s
}else if(s.attribute.contains("市重点")){
level2 << s
}else if(s.attribute.contains("区重点")){
level3 << s
}else if(s.attribute.contains("普通")){
level4 << s
}else{
level5 << s
}
}else{
level5 << s
}
}
if(level0){
if(level0.size()==1){
minSchool = level0.get(0)
}else{
minSchool = selectMinDistanceSchool(lat, lng, level0)
}
}else if(level1){
if(level1.size()==1){
minSchool = level1.get(0)
}else{
minSchool = selectMinDistanceSchool(lat, lng, level1)
}
}else if(level2){
if(level2.size()==1){
minSchool = level2.get(0)
}else{
minSchool = selectMinDistanceSchool(lat, lng, level2)
}
}else if(level3){
if(level3.size()==1){
minSchool = level3.get(0)
}else{
minSchool = selectMinDistanceSchool(lat, lng, level3)
}
}else if(level4){
if(level4.size()==1){
minSchool = level4.get(0)
}else{
minSchool = selectMinDistanceSchool(lat, lng, level4)
}
}else if(level5){
if(level5.size()==1){
minSchool = level5.get(0)
}else{
minSchool = selectMinDistanceSchool(lat, lng, level5)
}
}
def rets = []
if(minSchool){
rets = findSchoolInfo(minSchool.city, minSchool.match_region,minSchool.school_name)
}
if(rets.size()>0){
for(def d : rets){
if(!minSchool.school_address){
minSchool.school_address = d.address
}
if(!minSchool.attribute){
minSchool.attribute = d.attribute
}
}
}else{
if(minSchool){
if(minSchool.school_map_lng){
minSchool.school_lng2 = minSchool.school_map_lng
minSchool.school_lat2 = minSchool.school_map_lat
}else if(minSchool.school_lng1){
minSchool.school_lng2 = minSchool.school_lng1
minSchool.school_lat2 = minSchool.school_lat1
}else if(minSchool.school_lng){
minSchool.school_lng2 = minSchool.school_lng
minSchool.school_lat2 = minSchool.school_lat
}
}
}
return minSchool;
}
def findSchoolInfo(def city, def region, def schoolName){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", schoolName)
def dbcursor = school_col.find(q)
def rets = []
for(def obj : dbcursor){
rets << obj
}
return rets
}
def selectMinDistanceSchool(def lat, def lng, def schools){
def minSchool = null;
def minDistance = Integer.MAX_VALUE
for(def s1 : schools){
def lng2 = null
def lat2 = null
if(s1.school_map_lng){
lng2 = s1.school_map_lng
lat2 = s1.school_map_lat
}else if(s1.school_lng1){
lng2 = s1.school_lng1
lat2 = s1.school_lat1
}else if(s1.school_lng){
lng2 = s1.school_lng
lat2 = s1.school_lat
}
if(lat2){
def distance = GeoHelper.GetDistance(lat, lng, lat2, lng2)
println s1.district_name +" "+ s1.school_name+" "+distance
if(distance<minDistance){
minDistance = distance
minSchool = s1
minSchool.school_lng2 = lng2
minSchool.school_lat2 = lat2
}
}
}
return minSchool
}
def findPrimarySchool(def city, def region, def name){
def q = new BasicDBObject()
q.put("match_city", city)
q.put("match_region", region)
q.put("match_name", name)
q.put("category", Pattern.compile("^.*小学.*\$", Pattern.CASE_INSENSITIVE))
def dbcursor = district_school_col.find(q).sort(new BasicDBObject("attribute", 1))
def datas = []
for(def db : dbcursor){
datas << db
}
return datas;
}
def findMiddleSchool(def city, def region, def name){
def q = new BasicDBObject()
q.put("match_city", city)
q.put("match_region", region)
q.put("match_name", name)
q.put("category", new BasicDBObject(QueryOperators.NE, "小学"))
def dbcursor = district_school_col.find(q).sort(new BasicDBObject("attribute", 1))
def datas = []
for(def db : dbcursor){
datas << db
}
return datas;
}
def getRegionCodeByCity(def city){
def datas = FangJiaAreaMapUtils.convertToOfficials(city);
for(def d : datas){
regionCodeMap.put(city+"-"+d.fangjia_district, d.district_code)
}
}
}
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"]
for(def source :sources){
def count1 = findTotalCount(source, 202101);
def count2 = findTotalCount(source, 202102);
def count1 = findTotalCount(source, 202102);
def count2 = findTotalCount(source, 202103);
println source +" "+count1 +" "+count2
}
}
@@ -0,0 +1,213 @@
package com.fangjia.script.thirdPriceCrawl.other
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import net.sf.json.groovy.JsonSlurper
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
class AddFangSchoolRegion {
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
static DBCollection third_price_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "third_party_price")
static DBCollection school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "school2")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
def task = new AddFangSchoolRegion();
//task.process()
//task.match()
task.addMapLocation()
}
def addMapLocation(){
def q = new BasicDBObject()
def dbcursor = district_school_col.find(q)
def index = 0
for(def d : dbcursor){
def data = findSchool(d.city, d.region, d.school_name)
if(data && data.location){
def location = data.location
if(location){
def udata = new BasicDBObject()
udata.put("school_map_lng", location[0])
udata.put("school_map_lat", location[1])
district_school_col.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', udata))
}
}
index ++
println index
}
}
def match(){
def datas = findAllDatas()
final CountDownLatch latch = new CountDownLatch(datas.size())
for(def i=0;i<datas.size;i++){
def data = datas[i]
fixedThreadPool.execute(
new Runnable() {
void run() {
try {
// if(!data.match_name){
println i +" "+data.district_name
matching(data);
// }
} catch (Exception e) {
println "Òì³£"
e.printStackTrace();
} finally {
latch.countDown()
}
}
}
);
}
latch.await()
}
def matching(def it){
def rMap = matchDistrict(it.city, it.region, it.district_name)
if(!rMap){
rMap = matchDistrict(it.city, it.region, it.address)
}
if(rMap){
DBObject data = new BasicDBObject()
data.put("match_city", rMap.city)
data.put("match_region", rMap.region)
data.put("match_name", rMap.name)
data.put("match_address", rMap.address)
data.put("match_lat", rMap.lat)
data.put("match_lng", rMap.lng)
district_school_col.update(new BasicDBObject('_id', it.get("_id")), new BasicDBObject('$set', data))
}
}
def matchDistrict(def city, def region, def address){
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&preciseLevel=strict&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district"
String content = new HttpRequest().get(url, "utf-8");
def rMap = [:]
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==200 && doc.result){
def dto = doc.result
if(dto && dto.searchAddress){
def ret = dto.matchResult
if(ret.match){
rMap.city = dto.city
rMap.region = dto.district
rMap.name = dto.name
rMap.lat = dto.lat
rMap.lng = dto.lng
rMap.id = dto.id
rMap.address = dto.address
}
}
}
}
return rMap;
}
def process(){
def datas = findNoRegionData();
findSimialData(datas);
}
def findAllDatas(){
def q = new BasicDBObject()
q.put("match_name", new BasicDBObject(QueryOperators.EXISTS, false))
def dbcursor = district_school_col.find(q)
def rets =[]
for(def d : dbcursor){
rets << d
}
return rets;
}
def findNoRegionData(){
def q = new BasicDBObject()
q.put("region", null)
def dbcursor = district_school_col.find(q)
def rets = []
for(def d : dbcursor){
rets << d
}
return rets;
}
def findSimialData(datas){
for(def d : datas){
def obj = findThirdPriceData(d.city, d.url+"/")
if(!obj){
obj = findThirdPriceByName(d.city, d.district_name)
}
if(obj){
d.region = obj.region
if(!d.district_addrss){
d.district_address = obj.address
}
updateDistrictSchool(d);
}else{
println d.school_name+"" + d.district_name+"û²éµ½"
}
}
}
def findThirdPriceData(def city, def url){
def q = new BasicDBObject()
q.put("source","ËÑ·¿")
q.put("city", city)
q.put("district_id", url)
def dbcursor = third_price_col.find(q).sort(new BasicDBObject("s_date", -1)).limit(1)
def obj = null;
for(def d : dbcursor){
obj = d
}
return obj;
}
def findThirdPriceByName(def city, def name){
def q = new BasicDBObject()
q.put("source","ËÑ·¿")
q.put("city", city)
q.put("name", name)
def dbcursor = third_price_col.find(q).sort(new BasicDBObject("s_date", -1)).limit(1)
def obj = null;
for(def d : dbcursor){
obj = d
}
return obj;
}
def updateDistrictSchool(d){
DBObject data = new BasicDBObject()
data.put("region", d.region)
data.put("district_address", d.district_address)
district_school_col.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', data))
}
def findSchool(city, region, school_name){
DBObject data = new BasicDBObject()
data.put("city", city)
data.put("region", region)
data.put("name", school_name)
return school_col.findOne(data)
}
}
@@ -0,0 +1,344 @@
package com.fangjia.script.thirdPriceCrawl.other
import com.alibaba.fastjson.annotation.JSONField
import com.fangjia.core.RegionData
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.HttpClient
import com.fangjia.script.util.JacksonUtil
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.commons.httpclient.NameValuePair
import org.apache.commons.lang.StringUtils
import org.apache.http.HttpEntity
import org.apache.http.client.entity.UrlEncodedFormEntity
import org.apache.http.client.methods.CloseableHttpResponse
import org.apache.http.client.methods.HttpPost
import org.apache.http.impl.client.CloseableHttpClient
import org.apache.http.impl.client.HttpClients
import org.apache.http.message.BasicNameValuePair
import org.apache.http.util.EntityUtils
import java.nio.charset.Charset
import java.util.zip.GZIPInputStream
class GushuwangCrawl {
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
static DBCollection school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "school")
static main(args) {
def task = new GushuwangCrawl();
task.process()
}
void process(Object args) {
// chaxuequCrawl()
suzhouCrawl();
}
def suzhouCrawl(){
def page = 1
def stop = false
def list = []
while(!stop){
def s = schoolList(page);
if(s){
JSONObject jsonObject = JSONObject.fromObject(s);
if(jsonObject && jsonObject.ok){
def datas = jsonObject.extra.list
if(datas){
list.addAll(datas)
page++
println page
}else{
stop = true
}
}else{
stop = true
}
}
}
if(list){
def schools = []
for(def d : list){
def schoolData = new BasicDBObject()
schoolData.province = "江苏"
schoolData.city = "苏州"
schoolData.region = RegionData.parseRegionFormAddress(schoolData.city , d.adr)
/**类型: 幼儿园,小学,初中*/
schoolData.category = converCategory(d.schoolname)
schoolData.name = d.schoolname
schoolData.address = d.adr
/**来源网站*/
schoolData.source = "苏州学区查询"
/**房源详情url*/
schools << schoolData
}
saveChaxuequSchool(schools)
}
}
def chaxuequCrawl(){
def page = 1
def stop = false
def schools = []
while(!stop){
def s = chaxuequInfoCrawl("https://51yongka.com/fapi/school/list", "schoolName=&city=%E8%8B%8F%E5%B7%9E&area=&schoolNature=&schoolType=&page="+page+"&size=10")
println "内容:"+s
if(s){
JSONObject jsonObject = JSONObject.fromObject(s);
if(jsonObject.code == 200){
def data = jsonObject.data
def totalPage = data.pages
if(data.list){
page++
if(page+1>totalPage){
stop = true;
}
schools.addAll(data.list)
}
}
}
}
print(schools.size());
def districtSchools = []
def schoolInfos = []
if(schools.size()>0){
def url = "https://51yongka.com/fapi/school/details"
for(def d : schools){
def params = "schoolId="+d.id+"&access_token=oRqu15Mq_KxHfK0y0ZWhwktVnfUM"
def s = chaxuequInfoCrawl(url, params)
JSONObject jsonObject = JSONObject.fromObject(s);
if(jsonObject && jsonObject.code==200){
if(jsonObject.data && jsonObject.data.nearbyPropertyList){
def districts = jsonObject.data.nearbyPropertyList
for(def dist : districts){
def dataMap = new BasicDBObject()
dataMap.city = jsonObject.data.city
dataMap.region = RegionData.formatRegion(dataMap.city , jsonObject.data.area)
/**类型: 幼儿园,小学,初中*/
dataMap.category = converCategory(jsonObject.data.schoolName)
dataMap.district_name = dist.name.trim()
dataMap.district_address = dist.address
dataMap.school_name = jsonObject.data.schoolName.trim()
dataMap.school_address = jsonObject.data.address
dataMap.source = "查学区房"
dataMap.url = url + "?"+params
dataMap.telephone = jsonObject.data.officialTelephone;
districtSchools << dataMap
}
}else{
def schoolData = new BasicDBObject()
schoolData.province = "江苏"
schoolData.city = jsonObject.data.city
schoolData.region = RegionData.formatRegion(schoolData.city , jsonObject.data.area)
/**类型: 幼儿园,小学,初中*/
schoolData.category = converCategory(jsonObject.data.schoolName)
schoolData.name = jsonObject.data.schoolName.trim()
schoolData.address = jsonObject.data.address
/**来源网站*/
schoolData.source = "查学区房"
/**房源详情url*/
schoolData.telephone = jsonObject.data.officialTelephone;
schoolInfos << schoolData
println jsonObject.data.schoolName +" 无周边楼盘"
}
}
println s;
}
}
println "小区:"+districtSchools.size()
// saveChaxuequDistrictSchool(districtSchools)
saveChaxuequSchool(schoolInfos)
}
def saveChaxuequSchool(schoolInfos){
for(def ds : schoolInfos){
def obj = findSchoolData(ds.city, ds.school_name);
if(!obj){
school_col.save(ds)
}
}
}
def saveChaxuequDistrictSchool(def districtSchools){
for(def ds : districtSchools){
def obj = findDistrictSchoolData(ds.city, ds.school_name);
if(!obj){
district_school_col.save(ds)
}
}
}
def findSchoolData(city, school_name){
def q = new BasicDBObject()
q.put("city", city)
q.put("school_name", school_name)
return school_col.findOne(q)
}
def findDistrictSchoolData(city, school_name, district_name){
def q = new BasicDBObject()
q.put("city", city)
q.put("school_name", school_name)
q.put("district_name", district_name)
return district_school_col.findOne(q)
}
def converCategory(def schoolName){
if(schoolName){
if(schoolName.contains("初中部")||schoolName.contains("中学")){
return "初中"
}else if(schoolName.contains("小学")){
return "小学"
}
}
return null
}
def static chaxuequInfoCrawl(url, bodyquery){
def responseMap = [:]
try {
//headers信息
Map<String, String> headers = new LinkedHashMap<String, String>()
headers.put("Host","51yongka.com")
headers.put("Connection","keep-alive")
headers.put("Content-Length","75")
headers.put("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat")
headers.put("content-type","application/x-www-form-urlencoded")
headers.put("Referer","https://servicewechat.com/wxc2461b2529576950/8/page-frame.html")
headers.put("Accept-Encoding","gzip, deflate, br")
String s = doPost(url, bodyquery, headers)
return s
} catch (Exception e) {
e.printStackTrace()
}
return null
}
def static schoolList(page) {
def responseMap = [:]
String s = null
try {
//headers信息
Map<String, String> headers = new LinkedHashMap<String, String>()
headers.put("Host","m.gusuwang.com")
headers.put("Connection","keep-alive")
headers.put("Content-Length","40")
headers.put("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat")
headers.put("content-type","application/x-www-form-urlencoded")
headers.put("Referer","https://servicewechat.com/wx9188a941e5ef95c5/10/page-frame.html")
// headers.put("Accept-Encoding","gzip, deflate, br")
//params信息
Map<String, String> params = new HashMap<String, String>();
params.put("secCode","198a53da6baa3aa6a5b99959a006be37");
// //post请求
def url = "https://m.gusuwang.com/xuequ/xqapi.php?act=searchschool"
s = doPost(url, "secCode=198a53da6baa3aa6a5b99959a006be37&keys=&grade=&lb=&lx=&area=&currpage="+page,headers)
} catch (Exception e) {
e.printStackTrace()
}
return s
}
static String doPost(String strUrl, String content, Map headers) {
String result = "";
OutputStream outputStream = null;
InputStream inputStream = null;
StringBuilder sb = new StringBuilder();
try {
URL url = new URL(strUrl);
//通过调用url.openConnection()来获得一个新的URLConnection对象,并且将其结果强制转换为HttpURLConnection.
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
urlConnection.setRequestMethod("POST");
//设置连接的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
urlConnection.setConnectTimeout(30000);
//设置读取的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
urlConnection.setReadTimeout(30000);
//将url连接用于输出,这样才能使用getOutputStream()。getOutputStream()返回的输出流用于传输数据
urlConnection.setDoOutput(true);
urlConnection.setDoInput(true);
//设置通用请求属性为默认浏览器编码类型
// urlConnection.setRequestProperty("content-type", "application/x-www-form-urlencoded");
if (headers != null && headers.keySet().size() > 0) {
for (String head : headers.keySet()) {
urlConnection.setRequestProperty(head, headers.get(head));
}
}
//getOutputStream()返回的输出流,用于写入参数数据。
outputStream = urlConnection.getOutputStream();
outputStream.write(content.getBytes());
outputStream.flush();
outputStream.close();
//此时将调用接口方法。getInputStream()返回的输入流可以读取返回的数据。
inputStream = urlConnection.getInputStream();
// byte[] data = new byte[1024];
// //inputStream每次就会将读取1024个byte到data中,当inputSteam中没有数据时,inputStream.read(data)值为-1
// while (inputStream.read(data) != -1) {
// String s = new String(data, Charset.forName("utf-8"));
// sb.append(s);
// }
InputStreamReader reader = new InputStreamReader(inputStream, "UTF-8");
char[] buff = new char[1024];
int length = 0;
while ((length = reader.read(buff)) != -1) {
String s = new String(buff, 0, length);
sb.append(s);
}
} catch (IOException e) {
e.printStackTrace();
}finally{
if(outputStream!=null){
outputStream.close();
}
if(inputStream!=null){
inputStream.close();
}
}
return sb.toString();
}
}
@@ -52,7 +52,7 @@ class ElasticUtils {
return dataSource;
}
static executeSqlQuery(String sql, Closure<Map> closure, String host = "192.168.0.216"){
static executeSqlQuery(String sql, Closure<Map> closure, String host = "192.168.0.206"){
EsDataSource dataSource = getEsDataSource(host)
Connection connection = dataSource.getConnection()
Statement statement = connection.createStatement()