diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy index 01d0e126d..517fcf4ed 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy @@ -1,11 +1,15 @@ package com.fangjia.script.temp.shenjian +import com.fangjia.script.temp.shenjian.util.GaoDeUtils + /** * @author shenjian* @since 2021/5/18 */ class Temp { static void main(String[] args) { - + // 121.431870,31.208610 + List list = GaoDeUtils.searchPoi("上海", "浦东新区新场实验小学新城校区") + print(list) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy index 73545a2d8..b988b6a9d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy @@ -44,10 +44,10 @@ class SeaweedSchool { Map headers = headerMap("") HttpRequest re = new HttpRequest(); String str = null - final String area = "浦东新区"; + final String area = "浦东"; final int perPage = 10 final int total = 4733; - for (int page = 15; page < total / perPage; page++) { + for (int page = 1; page < total / perPage; page++) { Thread.sleep(1000) print("当前页:" + page) try { @@ -71,16 +71,18 @@ class SeaweedSchool { List listDTOS = houseList.getContent().getList(); if (CollectionUtils.isNotEmpty(listDTOS)) { for (HouseList.ContentDTO.ListDTO listDTO : listDTOS) { - Thread.sleep(1000) + Thread.sleep(500) Integer id = listDTO.getId(); //# 房屋详情 HouseDetail houseDetail = getHouseDetail(re, id, headers) + // 对口幼儿园 + School nurserySchool = getNurserySchool(re, id, headers) //# 对口小学 School primarySchool = getPrimarySchool(re, id, headers) //# 对口中学 School middleSchool = getMiddleSchool(re, id, headers) String url = "https://mapmapping.com/school-web/house/getHouseInfoDetail?source=1&id=" + String.valueOf(id) - saveData(houseDetail, primarySchool, middleSchool, url); + saveData(houseDetail, nurserySchool, primarySchool, middleSchool, url); } } } @@ -88,7 +90,7 @@ class SeaweedSchool { } } - private static void saveData(HouseDetail houseDetail, School primarySchool, School middleSchool, def url) { + private static void saveData(HouseDetail houseDetail, School nurserySchool, School primarySchool, School middleSchool, def url) { DBObject dbObject = new BasicDBObject() dbObject.put("url", url) dbObject.put("root", "沪上学区房") @@ -97,38 +99,60 @@ class SeaweedSchool { dbObject.put("region", houseDetail.getContent().getArea()) dbObject.put("name", houseDetail.getContent().getName()) dbObject.put("address", houseDetail.getContent().getAddress()) - List primarySchools = primarySchool.getContent(); + dbObject.put("lng", houseDetail.getContent().getLng()) + dbObject.put("lat", houseDetail.getContent().getLat()) + + List nurserySchools = nurserySchool.getContent().getList(); + if (CollectionUtils.isNotEmpty(nurserySchools)) { + List schools = buildSchool(nurserySchools) + dbObject.put("nurserySchool", schools) + } + + List primarySchools = primarySchool.getContent().getList();; if (CollectionUtils.isNotEmpty(primarySchools)) { - List schools = new ArrayList<>() - for (School.ContentDTO contentDTO : primarySchools) { - DBObject schoolObject = new BasicDBObject(); - if ("小学".equals(contentDTO.getType())) { - schoolObject.put("name", contentDTO.getName()) - schoolObject.put("quality", contentDTO.getQuality()) - schools.add(schoolObject) - } - } + List schools = buildSchool(primarySchools) dbObject.put("primarySchool", schools) } - List middleSchools = middleSchool.getContent(); + List middleSchools = middleSchool.getContent().getList();; if (CollectionUtils.isNotEmpty(middleSchools)) { - List schools = new ArrayList<>() - for (School.ContentDTO contentDTO : middleSchools) { - DBObject schoolObject = new BasicDBObject(); - if ("初中".equals(contentDTO.getType())) { - schoolObject.put("name", contentDTO.getName()) - schoolObject.put("quality", contentDTO.getQuality()) - schools.add(schoolObject) - } - } + List schools = buildSchool(middleSchools) dbObject.put("middleSchool", schools) } - collection.update(new BasicDBObject("url", url).append("root", "沪上学区房"), dbObject, true, false) } + private static List buildSchool(List currSchools) { + List schools = new ArrayList<>() + for (School.ContentDTO.ListDTO contentDTO : currSchools) { + DBObject schoolObject = new BasicDBObject(); + if (com.fangjia.script.coredata.util.CollectionUtils.isNotEmpty(contentDTO.getTags())) { + if (contentDTO.getTags().size() >= 3) { + schoolObject.put("quality", contentDTO.getTags().get(0)) + schoolObject.put("type", contentDTO.getTags().get(1)) + schoolObject.put("batch", contentDTO.getTags().get(2)) + } else if (contentDTO.getTags().size() == 2) { + schoolObject.put("quality", contentDTO.getTags().get(0)) + schoolObject.put("type", contentDTO.getTags().get(1)) + schoolObject.put("batch", '') + } else { + schoolObject.put("quality", contentDTO.getTags().get(0)) + schoolObject.put("type", '') + schoolObject.put("batch", '') + } + } + schoolObject.put("area", contentDTO.getArea()) + schoolObject.put("town", contentDTO.getTown()) + schoolObject.put("distance", contentDTO.getDistance()) + schoolObject.put("address", contentDTO.getAddress()) + schools.add(schoolObject) + } + return schools + } + + + private static HouseDetail getHouseDetail(def re, def id, def headers) { Map params = new HashMap(); params.put("id", String.valueOf(id)) @@ -137,6 +161,17 @@ class SeaweedSchool { return JSONObject.parseObject(str, HouseDetail.class) } + private static School getNurserySchool(def re, def id, def headers) { + Map params = new HashMap(); + params.put("id", String.valueOf(id)) + params.put("type", "1") + params.put("page", "1") + params.put("perPage", "2") + params.put("source", "1") + def str = re.get("https://mapmapping.com/school-web/school/getPlotSchoolList", params, headers, "UTF-8"); + return JSONObject.parseObject(str, School.class) + } + private static School getPrimarySchool(def re, def id, def headers) { Map params = new HashMap(); params.put("id", String.valueOf(id)) @@ -144,7 +179,7 @@ class SeaweedSchool { params.put("page", "1") params.put("perPage", "2") params.put("source", "1") - def str = re.get("https://mapmapping.com/school-web/house/getPlotSchoolList", params, headers, "UTF-8"); + def str = re.get("https://mapmapping.com/school-web/school/getPlotSchoolList", params, headers, "UTF-8"); return JSONObject.parseObject(str, School.class) } @@ -155,7 +190,7 @@ class SeaweedSchool { params.put("page", "1") params.put("perPage", "4") params.put("source", "1") - def str = re.get("https://mapmapping.com/school-web/house/getPlotSchoolList", params, headers, "UTF-8"); + def str = re.get("https://mapmapping.com/school-web/school/getPlotSchoolList", params, headers, "UTF-8"); return JSONObject.parseObject(str, School.class) } } diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/School.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/School.java index 1beb5153b..992727e92 100644 --- a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/School.java +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/School.java @@ -1,6 +1,6 @@ package com.fangjia.script.statistics.seaweed; -import com.alibaba.fastjson.annotation.JSONField; +import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; import lombok.NoArgsConstructor; @@ -15,71 +15,52 @@ import java.util.List; public class School { - @JSONField(name = "code") + @JsonProperty("code") private String code; - @JSONField(name = "desc") + @JsonProperty("desc") private String desc; - @JSONField(name = "content") - private List content; - @JSONField(name = "timestamp") + @JsonProperty("content") + private ContentDTO content; + @JsonProperty("timestamp") private Long timestamp; @NoArgsConstructor @Data public static class ContentDTO { - @JSONField(name = "id") - private Integer id; - @JSONField(name = "name") - private String name; - @JSONField(name = "nature") - private Object nature; - @JSONField(name = "city") - private String city; - @JSONField(name = "area") - private String area; - @JSONField(name = "type") - private String type; - @JSONField(name = "quality") - private String quality; - @JSONField(name = "location") - private Object location; - @JSONField(name = "picture") - private String picture; - @JSONField(name = "phone") - private String phone; - @JSONField(name = "distanceNum") - private Object distanceNum; - @JSONField(name = "plotNum") - private String plotNum; - @JSONField(name = "sellNum") - private String sellNum; - @JSONField(name = "plotPrice") - private String plotPrice; - @JSONField(name = "address") - private Object address; - @JSONField(name = "lat") - private Object lat; - @JSONField(name = "lng") - private Object lng; - @JSONField(name = "scope") - private Object scope; - @JSONField(name = "score") - private Object score; - @JSONField(name = "scopeArr") - private Object scopeArr; - @JSONField(name = "scopeTag") - private Object scopeTag; - @JSONField(name = "tag") - private Object tag; - @JSONField(name = "shortName") - private Object shortName; - @JSONField(name = "view") - private Object view; - @JSONField(name = "construct") - private Object construct; - @JSONField(name = "echelon") - private Object echelon; - @JSONField(name = "town") - private Object town; + @JsonProperty("total") + private Integer total; + @JsonProperty("hasNextPage") + private Boolean hasNextPage; + @JsonProperty("list") + private List list; + + @NoArgsConstructor + @Data + public static class ListDTO { + @JsonProperty("id") + private Integer id; + @JsonProperty("name") + private String name; + @JsonProperty("area") + private String area; + @JsonProperty("shortName") + private Object shortName; + @JsonProperty("picture") + private String picture; + @JsonProperty("distance") + private String distance; + @JsonProperty("plotPrice") + private String plotPrice; + @JsonProperty("topInfo") + private Object topInfo; + @JsonProperty("view") + private String view; + @JsonProperty("town") + private String town; + @JsonProperty("address") + private String address; + @JsonProperty("tags") + private List tags; + } } }