新增浦东对口小学初中抓取

This commit is contained in:
沈健
2021-09-02 19:08:26 +08:00
parent 3f7209bb47
commit ca55e1a30e
7 changed files with 792 additions and 0 deletions
@@ -0,0 +1,159 @@
package com.fangjia.script.temp.shenjian.seaweed
import com.alibaba.fastjson.JSONObject
import com.fangjia.core.Constants
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.script.statistics.seaweed.HouseDetail
import com.fangjia.script.statistics.seaweed.HouseList
import com.fangjia.script.statistics.seaweed.School
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.client.MongoCollection
import org.apache.commons.collections.CollectionUtils
import org.apache.commons.lang.StringUtils
import org.bson.Document
import org.jdom.Content;
/**
* @author shenjian* @since 2021/9/2
*/
class SeaweedSchool {
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "seaweed_school", "192.168.0.235", 27018)
public static Map<String, String> headerMap(String cook) {
Map<String, String> headers = new HashMap<String, String>();
headers.put("Host", "www.mapmapping.com")
headers.put("Connection", "keep-alive")
headers.put("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat")
headers.put("Content-Type", "application/json")
headers.put("Cookie", "_applet_id=34185;_applet_ticket=e294e2426345401088a3e828128e70e3")
headers.put("Referer", "https://servicewechat.com/wx5568ec264e167aeb/9/page-frame.html")
headers.put("Accept-Encoding", "gzip, deflate, br")
return headers;
}
static void main(String[] args) {
Map<String, String> headers = headerMap("")
HttpRequest re = new HttpRequest();
String str = null
final String area = "浦东新区";
final int perPage = 10
final int total = 4733;
for (int page = 1; page < total / perPage; page++) {
Thread.sleep(10000)
print("当前页:" + page)
try {
Map<String, String> params = new HashMap<String, String>();
params.put("city", "上海市")
params.put("area", area)
params.put("page", String.valueOf(page))
params.put("perPage", String.valueOf(perPage))
params.put("priceType", "")
params.put("order", "")
params.put("source", "1")
str = re.get("https://mapmapping.com/school-web/house/getHouseInfoList", params, headers, "UTF-8");
} catch (Exception e) {
e.printStackTrace()
}
println str
if (StringUtils.isNotEmpty(str)) {
HouseList houseList = JSONObject.parseObject(str, HouseList.class);
if (houseList != null && "000".equals(houseList.getCode())) {
/*Integer total = houseList.getContent().getTotal();*/
List<HouseList.ContentDTO.ListDTO> listDTOS = houseList.getContent().getList();
if (CollectionUtils.isNotEmpty(listDTOS)) {
for (HouseList.ContentDTO.ListDTO listDTO : listDTOS) {
Thread.sleep(5000)
Integer id = listDTO.getId();
//# 房屋详情
HouseDetail houseDetail = getHouseDetail(re, id, headers)
//# 对口小学
School primarySchool = getPrimarySchool(re, id, headers)
//# 对口中学
School middleSchool = getMiddleSchool(re, id, headers)
String url = "https://mapmapping.com/school-web/house/getHouseInfoDetail?source=1&id=" + String.valueOf(id)
saveData(houseDetail, primarySchool, middleSchool, url);
}
}
}
}
}
}
private static void saveData(HouseDetail houseDetail, School primarySchool, School middleSchool, def url) {
DBObject dbObject = new BasicDBObject()
dbObject.put("url", url)
dbObject.put("root", "沪上学区房")
dbObject.put("c_date", new Date())
dbObject.put("city", "上海")
dbObject.put("region", houseDetail.getContent().getArea())
dbObject.put("name", houseDetail.getContent().getName())
dbObject.put("address", houseDetail.getContent().getAddress())
List<School.ContentDTO> primarySchools = primarySchool.getContent();
if (CollectionUtils.isNotEmpty(primarySchools)) {
List<DBObject> schools = new ArrayList<>()
for (School.ContentDTO contentDTO : primarySchools) {
DBObject schoolObject = new BasicDBObject();
if ("小学".equals(contentDTO.getType())) {
schoolObject.put("name", contentDTO.getName())
schoolObject.put("quality", contentDTO.getQuality())
schools.add(schoolObject)
}
}
dbObject.put("primarySchool", schools)
}
List<School.ContentDTO> middleSchools = middleSchool.getContent();
if (CollectionUtils.isNotEmpty(middleSchools)) {
List<DBObject> schools = new ArrayList<>()
for (School.ContentDTO contentDTO : middleSchools) {
DBObject schoolObject = new BasicDBObject();
if ("初中".equals(contentDTO.getType())) {
schoolObject.put("name", contentDTO.getName())
schoolObject.put("quality", contentDTO.getQuality())
schools.add(schoolObject)
}
}
dbObject.put("middleSchool", schools)
}
collection.update(new BasicDBObject("url", url).append("root", "沪上学区房"), dbObject, true, false)
}
private static HouseDetail getHouseDetail(def re, def id, def headers) {
Map<String, String> params = new HashMap<String, String>();
params.put("id", String.valueOf(id))
params.put("source", "1")
def str = re.get("https://mapmapping.com/school-web/house/getHouseInfoDetail", params, headers, "UTF-8");
return JSONObject.parseObject(str, HouseDetail.class)
}
private static School getPrimarySchool(def re, def id, def headers) {
Map<String, String> params = new HashMap<String, String>();
params.put("id", String.valueOf(id))
params.put("type", "2")
params.put("page", "1")
params.put("perPage", "2")
params.put("source", "1")
def str = re.get("https://mapmapping.com/school-web/house/getPlotSchoolList", params, headers, "UTF-8");
return JSONObject.parseObject(str, School.class)
}
private static School getMiddleSchool(def re, def id, def headers) {
Map<String, String> params = new HashMap<String, String>();
params.put("id", String.valueOf(id))
params.put("type", "3")
params.put("page", "1")
params.put("perPage", "4")
params.put("source", "1")
def str = re.get("https://mapmapping.com/school-web/house/getPlotSchoolList", params, headers, "UTF-8");
return JSONObject.parseObject(str, School.class)
}
}
File diff suppressed because one or more lines are too long
@@ -0,0 +1,255 @@
{
"code": "000",
"desc": "操作成功",
"content": {
"total": 4733,
"list": [
{
"id": 59151,
"name": "凌高小区",
"area": "浦东",
"img": "https://img.mapmapping.com/a68e7f60-a8d8-4e25-ade8-2d77c86562cd.png?imageView2/2/w/240/h/200/q/75",
"town": "洋泾",
"schoolTag": "第六师范学校第二附属小学西校区",
"sellerNum": "11",
"leaseNum": "4",
"price": "6.64w",
"formatPrice": null,
"year": "1993年建成",
"lat": "31.25306680262166000",
"lng": "121.55601323149715000",
"tags": [
"近地铁6号线北洋泾路站"
],
"plotType": "塔楼",
"buildTime": "1993年建成",
"view": "1.53w",
"tag": "近地铁6号线北洋泾路站"
},
{
"id": 58324,
"name": "梅园四街坊",
"area": "浦东",
"img": "https://img.mapmapping.com/775871a1-e5a9-4e01-88ed-a39f040b51ac.png?imageView2/2/w/240/h/200/q/75",
"town": "梅园",
"schoolTag": "浦东新区福山外国语小学(福山/瑞华)",
"sellerNum": "15",
"leaseNum": "11",
"price": "13.42w",
"formatPrice": null,
"year": "1988年建成",
"lat": "31.23754371649258600",
"lng": "121.53281691712684000",
"tags": [
"近地铁2号线东昌路站"
],
"plotType": "塔楼/板楼/塔板结合",
"buildTime": "1988年建成",
"view": "1.52w",
"tag": "近地铁2号线东昌路站"
},
{
"id": 59165,
"name": "中星海上景庭",
"area": "浦东",
"img": "https://img.mapmapping.com/4f9d8c2a-8b89-49a8-b45f-ecaff4f3f588.png?imageView2/2/w/240/h/200/q/75",
"town": "洋泾",
"schoolTag": "第六师范学校第二附属小学西校区",
"sellerNum": "8",
"leaseNum": "3",
"price": "10.86w",
"formatPrice": null,
"year": "2004年建成",
"lat": "31.24523893361267700",
"lng": "121.55501162085352000",
"tags": [
"近地铁6号线北洋泾路站"
],
"plotType": "板楼",
"buildTime": "2004年建成",
"view": "1.50w",
"tag": "近地铁6号线北洋泾路站"
},
{
"id": 58477,
"name": "海怡别墅(公寓)",
"area": "浦东",
"img": "https://img.mapmapping.com/f50ed86d-474f-47c5-8f83-7c203c510d92.png?imageView2/2/w/240/h/200/q/75",
"town": "梅园",
"schoolTag": "浦东新区福山外国语小学(福山/瑞华)",
"sellerNum": "0",
"leaseNum": "0",
"price": "7.32w",
"formatPrice": null,
"year": "1994年建成",
"lat": "31.23963538757626600",
"lng": "121.53779361948267000",
"tags": [
"近地铁6号线源深体育中心站"
],
"plotType": "板楼",
"buildTime": "1994年建成",
"view": "1.50w",
"tag": "近地铁6号线源深体育中心站"
},
{
"id": 59148,
"name": "崮山小区一街坊",
"area": "浦东",
"img": "https://img.mapmapping.com/b651bbba-97ba-4ce0-831c-925e80b4459f.png?imageView2/2/w/240/h/200/q/75",
"town": "洋泾",
"schoolTag": "第六师范附属小学羽山校区",
"sellerNum": "10",
"leaseNum": "5",
"price": "11.81w",
"formatPrice": null,
"year": "1999年建成",
"lat": "31.24282189657667000",
"lng": "121.56992501943853000",
"tags": [
"近地铁9号线芳甸路站"
],
"plotType": "板楼",
"buildTime": "1999年建成",
"view": "1.50w",
"tag": "近地铁9号线芳甸路站"
},
{
"id": 55895,
"name": "证大家园(四期)",
"area": "浦东",
"img": "https://img.mapmapping.com/d7d0671f-b0ed-4082-a9f6-1883eca3ac85.png?imageView2/2/w/240/h/200/q/75",
"town": "高行",
"schoolTag": "浦东新区福山证大外国语小学",
"sellerNum": "11",
"leaseNum": "7",
"price": "9.63w",
"formatPrice": null,
"year": "2006年建成",
"lat": "31.28831283871331500",
"lng": "121.60787797808233000",
"tags": [
"近地铁6号线巨峰路站"
],
"plotType": "板楼",
"buildTime": "2006年建成",
"view": "1.49w",
"tag": "近地铁6号线巨峰路站"
},
{
"id": 58709,
"name": "新光大楼",
"area": "浦东",
"img": "https://img.mapmapping.com/f8e1dffb-fa85-457b-8a97-fb019239bdf6.png?imageView2/2/w/240/h/200/q/75",
"town": "潍坊",
"schoolTag": "浦东新区明珠小学A区",
"sellerNum": "3",
"leaseNum": "2",
"price": "11.95w",
"formatPrice": null,
"year": "1993年建成",
"lat": "31.23234474870607700",
"lng": "121.52462643697852000",
"tags": [
"近地铁9号线商城路站"
],
"plotType": "塔楼",
"buildTime": "1993年建成",
"view": "1.49w",
"tag": "近地铁9号线商城路站"
},
{
"id": 59322,
"name": "盛世年华",
"area": "浦东",
"img": "https://img.mapmapping.com/83727155-29f5-47d9-b54d-4042d9e625ec.png?imageView2/2/w/240/h/200/q/75",
"town": "源深",
"schoolTag": "浦东新区福山花园外国语小学",
"sellerNum": "4",
"leaseNum": "11",
"price": "14.94w",
"formatPrice": null,
"year": "2005年建成",
"lat": "31.23494392135101000",
"lng": "121.54545194344688000",
"tags": [
"近地铁6号线源深体育中心站"
],
"plotType": "板楼",
"buildTime": "2005年建成",
"view": "1.48w",
"tag": "近地铁6号线源深体育中心站"
},
{
"id": 58326,
"name": "梅园七街坊",
"area": "浦东",
"img": "https://img.mapmapping.com/30cf630f-830b-4e43-8dc5-648bb7dfb760.png?imageView2/2/w/240/h/200/q/75",
"town": "梅园",
"schoolTag": "浦东新区福山外国语小学(福山/瑞华)",
"sellerNum": "2",
"leaseNum": "2",
"price": "11.13w",
"formatPrice": null,
"year": "1986年建成",
"lat": "31.24378516838488700",
"lng": "121.53455423998314000",
"tags": [
"近地铁6号线源深体育中心站"
],
"plotType": "板楼",
"buildTime": "1986年建成",
"view": "1.48w",
"tag": "近地铁6号线源深体育中心站"
},
{
"id": 58599,
"name": "陈家门小区",
"area": "浦东",
"img": "https://img.mapmapping.com/7a9f43ef-67c4-4168-bf24-0190689d331d.png?imageView2/2/w/240/h/200/q/75",
"town": "梅园",
"schoolTag": "浦东新区梅园小学",
"sellerNum": "3",
"leaseNum": "1",
"price": "9.27w",
"formatPrice": null,
"year": "1986年建成",
"lat": "31.24726096884606800",
"lng": "121.53135690118148000",
"tags": [
"近地铁4号线浦东大道站"
],
"plotType": "板楼",
"buildTime": "1986年建成",
"view": "1.48w",
"tag": "近地铁4号线浦东大道站"
}
],
"pageNum": 1,
"pageSize": 10,
"size": 10,
"startRow": 1,
"endRow": 10,
"pages": 474,
"prePage": 0,
"nextPage": 2,
"isFirstPage": true,
"isLastPage": false,
"hasPreviousPage": false,
"hasNextPage": true,
"navigatePages": 8,
"navigatepageNums": [
1,
2,
3,
4,
5,
6,
7,
8
],
"navigateFirstPage": 1,
"navigateLastPage": 8
},
"timestamp": 1630575199849
}
@@ -0,0 +1,65 @@
{
"code": "000",
"desc": "操作成功",
"content": [
{
"id": 272113,
"name": "第六师范学校第二附属小学西校区",
"nature": null,
"city": "上海市",
"area": "浦东新区",
"type": "小学",
"quality": "公办",
"location": null,
"picture": "http://img.mapmapping.com//FpujwY5MscqLxCjjD27Mg-A4MY8d",
"phone": "",
"distanceNum": null,
"plotNum": "41",
"sellNum": "188",
"plotPrice": "8.69",
"address": null,
"lat": null,
"lng": null,
"scope": null,
"score": null,
"scopeArr": null,
"scopeTag": null,
"tag": null,
"shortName": null,
"view": null,
"construct": null,
"echelon": null,
"town": null
},
{
"id": 273287,
"name": "进才中学北校苗圃路校区",
"nature": null,
"city": "上海市",
"area": "浦东新区",
"type": "初中",
"quality": "公办",
"location": null,
"picture": "http://img.mapmapping.com//FpujwY5MscqLxCjjD27Mg-A4MY8d",
"phone": "",
"distanceNum": null,
"plotNum": "85",
"sellNum": "339",
"plotPrice": "9.33",
"address": null,
"lat": null,
"lng": null,
"scope": null,
"score": null,
"scopeArr": null,
"scopeTag": null,
"tag": null,
"shortName": null,
"view": null,
"construct": null,
"echelon": null,
"town": null
}
],
"timestamp": 1630576702404
}
@@ -0,0 +1,79 @@
package com.fangjia.script.statistics.seaweed;
import com.alibaba.fastjson.annotation.JSONField;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
/**
* @author shenjian
* @since 2021/9/2
*/
@NoArgsConstructor
@Data
public class HouseDetail {
@JSONField(name = "code")
private String code;
@JSONField(name = "desc")
private String desc;
@JSONField(name = "content")
private ContentDTO content;
@JSONField(name = "timestamp")
private Long timestamp;
@NoArgsConstructor
@Data
public static class ContentDTO {
@JSONField(name = "id")
private Integer id;
@JSONField(name = "name")
private String name;
@JSONField(name = "imgsUrl")
private String imgsUrl;
@JSONField(name = "area")
private String area;
@JSONField(name = "town")
private String town;
@JSONField(name = "price")
private String price;
@JSONField(name = "priceDesc")
private String priceDesc;
@JSONField(name = "plotType")
private String plotType;
@JSONField(name = "leaseNum")
private String leaseNum;
@JSONField(name = "sellNum")
private String sellNum;
@JSONField(name = "dealNum")
private String dealNum;
@JSONField(name = "buildTime")
private String buildTime;
@JSONField(name = "houseNum")
private String houseNum;
@JSONField(name = "building")
private String building;
@JSONField(name = "propertyFee")
private String propertyFee;
@JSONField(name = "propertyCompany")
private String propertyCompany;
@JSONField(name = "developers")
private String developers;
@JSONField(name = "lat")
private Double lat;
@JSONField(name = "lng")
private Double lng;
@JSONField(name = "imgArr")
private List<String> imgArr;
@JSONField(name = "focus")
private Boolean focus;
@JSONField(name = "tags")
private List<String> tags;
@JSONField(name = "view")
private Integer view;
@JSONField(name = "address")
private String address;
}
}
@@ -0,0 +1,107 @@
package com.fangjia.script.statistics.seaweed;
import com.alibaba.fastjson.annotation.JSONField;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
/**
* @author shenjian
* @since 2021/9/2
*/
@NoArgsConstructor
@Data
public class HouseList {
@JSONField(name = "code")
private String code;
@JSONField(name = "desc")
private String desc;
@JSONField(name = "content")
private ContentDTO content;
@JSONField(name = "timestamp")
private Long timestamp;
@NoArgsConstructor
@Data
public static class ContentDTO {
@JSONField(name = "total")
private Integer total;
@JSONField(name = "list")
private List<ListDTO> list;
@JSONField(name = "pageNum")
private Integer pageNum;
@JSONField(name = "pageSize")
private Integer pageSize;
@JSONField(name = "size")
private Integer size;
@JSONField(name = "startRow")
private Integer startRow;
@JSONField(name = "endRow")
private Integer endRow;
@JSONField(name = "pages")
private Integer pages;
@JSONField(name = "prePage")
private Integer prePage;
@JSONField(name = "nextPage")
private Integer nextPage;
@JSONField(name = "isFirstPage")
private Boolean isFirstPage;
@JSONField(name = "isLastPage")
private Boolean isLastPage;
@JSONField(name = "hasPreviousPage")
private Boolean hasPreviousPage;
@JSONField(name = "hasNextPage")
private Boolean hasNextPage;
@JSONField(name = "navigatePages")
private Integer navigatePages;
@JSONField(name = "navigatepageNums")
private List<Integer> navigatepageNums;
@JSONField(name = "navigateFirstPage")
private Integer navigateFirstPage;
@JSONField(name = "navigateLastPage")
private Integer navigateLastPage;
@NoArgsConstructor
@Data
public static class ListDTO {
@JSONField(name = "id")
private Integer id;
@JSONField(name = "name")
private String name;
@JSONField(name = "area")
private String area;
@JSONField(name = "img")
private String img;
@JSONField(name = "town")
private String town;
@JSONField(name = "schoolTag")
private String schoolTag;
@JSONField(name = "sellerNum")
private String sellerNum;
@JSONField(name = "leaseNum")
private String leaseNum;
@JSONField(name = "price")
private String price;
@JSONField(name = "formatPrice")
private Object formatPrice;
@JSONField(name = "year")
private String year;
@JSONField(name = "lat")
private String lat;
@JSONField(name = "lng")
private String lng;
@JSONField(name = "tags")
private List<String> tags;
@JSONField(name = "plotType")
private String plotType;
@JSONField(name = "buildTime")
private String buildTime;
@JSONField(name = "view")
private String view;
@JSONField(name = "tag")
private String tag;
}
}
}
@@ -0,0 +1,85 @@
package com.fangjia.script.statistics.seaweed;
import com.alibaba.fastjson.annotation.JSONField;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
/**
* @author shenjian
* @since 2021/9/2
*/
@NoArgsConstructor
@Data
public class School {
@JSONField(name = "code")
private String code;
@JSONField(name = "desc")
private String desc;
@JSONField(name = "content")
private List<ContentDTO> content;
@JSONField(name = "timestamp")
private Long timestamp;
@NoArgsConstructor
@Data
public static class ContentDTO {
@JSONField(name = "id")
private Integer id;
@JSONField(name = "name")
private String name;
@JSONField(name = "nature")
private Object nature;
@JSONField(name = "city")
private String city;
@JSONField(name = "area")
private String area;
@JSONField(name = "type")
private String type;
@JSONField(name = "quality")
private String quality;
@JSONField(name = "location")
private Object location;
@JSONField(name = "picture")
private String picture;
@JSONField(name = "phone")
private String phone;
@JSONField(name = "distanceNum")
private Object distanceNum;
@JSONField(name = "plotNum")
private String plotNum;
@JSONField(name = "sellNum")
private String sellNum;
@JSONField(name = "plotPrice")
private String plotPrice;
@JSONField(name = "address")
private Object address;
@JSONField(name = "lat")
private Object lat;
@JSONField(name = "lng")
private Object lng;
@JSONField(name = "scope")
private Object scope;
@JSONField(name = "score")
private Object score;
@JSONField(name = "scopeArr")
private Object scopeArr;
@JSONField(name = "scopeTag")
private Object scopeTag;
@JSONField(name = "tag")
private Object tag;
@JSONField(name = "shortName")
private Object shortName;
@JSONField(name = "view")
private Object view;
@JSONField(name = "construct")
private Object construct;
@JSONField(name = "echelon")
private Object echelon;
@JSONField(name = "town")
private Object town;
}
}