diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/BsEvaluationDao.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/BsEvaluationDao.java new file mode 100644 index 000000000..5f895886b --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/BsEvaluationDao.java @@ -0,0 +1,7 @@ +package com.fangjia.dal.mongo.dao; + +import com.fangjia.dal.MongoDao; +import com.fangjia.dal.mongo.model.BsEvaluation; + +public interface BsEvaluationDao extends MongoDao { +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/BsEvaluationDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/BsEvaluationDaoImpl.java new file mode 100644 index 000000000..e4d7fc080 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/BsEvaluationDaoImpl.java @@ -0,0 +1,8 @@ +package com.fangjia.dal.mongo.dao; + +import com.fangjia.dal.MongoDaoImpl; +import com.fangjia.dal.mongo.model.BsEvaluation; + +public class BsEvaluationDaoImpl extends MongoDaoImpl implements + BsEvaluationDao { +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/BsEvaluation.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/BsEvaluation.java new file mode 100644 index 000000000..d3b1b28f0 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/BsEvaluation.java @@ -0,0 +1,184 @@ +package com.fangjia.dal.mongo.model; + +import com.fangjia.dal.MongoEntity; + +import javax.persistence.*; +import java.util.Date; + +@Entity +@Table(name = "business.evaluation") +public class BsEvaluation implements MongoEntity { + private String id; + private String fullAddress; + private String commid; + private String cityName; + private String distName; + private String commName; + private String address; + private Double buildArea; + private Integer floor; + private Integer floorcnt; + private Integer roomcnt; + private Integer livingroomcnt; + private String houseaspect;//南 + private String fitment; + private Date cDate; + private Integer unitPrice; + private Integer totalPrice; + + @Id + @GeneratedValue(strategy = GenerationType.AUTO) + @Column(name = "_id") + public String getId() { + return id; + } + + @Override + public void setId(String id) { + this.id = id; + } + + @Column(name = "full_address") + public String getFullAddress() { + return fullAddress; + } + + public void setFullAddress(String fullAddress) { + this.fullAddress = fullAddress; + } + + @Column(name = "commid") + public String getCommid() { + return commid; + } + + public void setCommid(String commid) { + this.commid = commid; + } + + @Column(name = "cityname") + public String getCityName() { + return cityName; + } + + public void setCityName(String cityName) { + this.cityName = cityName; + } + + @Column(name = "distname") + public String getDistName() { + return distName; + } + + public void setDistName(String distName) { + this.distName = distName; + } + + @Column(name = "commname") + public String getCommName() { + return commName; + } + + public void setCommName(String commName) { + this.commName = commName; + } + + @Column(name = "address") + public String getAddress() { + return address; + } + + public void setAddress(String address) { + this.address = address; + } + + @Column(name = "buildarea") + public Double getBuildArea() { + return buildArea; + } + + public void setBuildArea(Double buildArea) { + this.buildArea = buildArea; + } + + @Column(name = "floor") + public Integer getFloor() { + return floor; + } + + public void setFloor(Integer floor) { + this.floor = floor; + } + + @Column(name = "floorcnt") + public Integer getFloorcnt() { + return floorcnt; + } + + public void setFloorcnt(Integer floorcnt) { + this.floorcnt = floorcnt; + } + + @Column(name = "roomcnt") + public Integer getRoomcnt() { + return roomcnt; + } + + public void setRoomcnt(Integer roomcnt) { + this.roomcnt = roomcnt; + } + + @Column(name = "livingroomcnt") + public Integer getLivingroomcnt() { + return livingroomcnt; + } + + public void setLivingroomcnt(Integer livingroomcnt) { + this.livingroomcnt = livingroomcnt; + } + + @Column(name = "houseaspect") + public String getHouseaspect() { + return houseaspect; + } + + public void setHouseaspect(String houseaspect) { + this.houseaspect = houseaspect; + } + + @Column(name = "fitment") + public String getFitment() { + return fitment; + } + + public void setFitment(String fitment) { + this.fitment = fitment; + } + + @Column(name = "c_date") + public Date getcDate() { + return cDate; + } + + public void setcDate(Date cDate) { + this.cDate = cDate; + } + + @Column(name = "unitprice") + public Integer getUnitPrice() { + return unitPrice; + } + + public void setUnitPrice(Integer unitPrice) { + this.unitPrice = unitPrice; + } + + @Column(name = "totalprice") + public Integer getTotalPrice() { + return totalPrice; + } + + public void setTotalPrice(Integer totalPrice) { + this.totalPrice = totalPrice; + } +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/DistrictFacilitys.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/DistrictFacilitys.java index 36021519c..cc946257b 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/DistrictFacilitys.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/DistrictFacilitys.java @@ -63,6 +63,7 @@ public class DistrictFacilitys implements MongoEntity { private String imgUrl3D; private List school; + private List keySchool; @Id @GeneratedValue @@ -380,4 +381,12 @@ public class DistrictFacilitys implements MongoEntity { this.school = school; } + @Column(name = "key_school") + public List getKeySchool() { + return keySchool; + } + + public void setKeySchool(List keySchool) { + this.keySchool = keySchool; + } } diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/DealPrice.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/DealPrice.java index a4ababe8e..a46cd7de0 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/DealPrice.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/DealPrice.java @@ -48,6 +48,11 @@ public class DealPrice implements MongoEntity { private Integer floor;//当前楼层 private Long score; + private String matchId; + private String matchCity; + private String matchRegion; + private String matchName; + @Id @GeneratedValue @Column(name = "_id", unique = true, nullable = false) @@ -296,4 +301,40 @@ public class DealPrice implements MongoEntity { } return true; } + + @Column(name = "match_id") + public String getMatchId() { + return matchId; + } + + public void setMatchId(String matchId) { + this.matchId = matchId; + } + + @Column(name = "match_city") + public String getMatchCity() { + return matchCity; + } + + public void setMatchCity(String matchCity) { + this.matchCity = matchCity; + } + + @Column(name = "match_region") + public String getMatchRegion() { + return matchRegion; + } + + public void setMatchRegion(String matchRegion) { + this.matchRegion = matchRegion; + } + + @Column(name = "match_name") + public String getMatchName() { + return matchName; + } + + public void setMatchName(String matchName) { + this.matchName = matchName; + } } diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/HouseLoudong.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/HouseLoudong.java index cf98a3665..d41b400ee 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/HouseLoudong.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/HouseLoudong.java @@ -448,6 +448,7 @@ public class HouseLoudong extends com.fangjia.dal.mongo.model.HouseLoudong{ this.openTotalPrice = openTotalPrice; } + @Column(name = "district_id") public String getDistrictId() { return districtId; } diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/Loudong.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/Loudong.java index 712b5fc7d..12cb8ca9b 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/Loudong.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/Loudong.java @@ -101,7 +101,11 @@ public class Loudong implements MongoEntity { * seaweed中的小区名 */ private String nameInSeaweed; - + /** + * seaweed中的小区id + */ + private String idInSeaweed; + @Column(name = "remark") public String getRemark() { return remark; @@ -402,4 +406,13 @@ public class Loudong implements MongoEntity { public void setNameInSeaweed(String nameInSeaweed) { this.nameInSeaweed = nameInSeaweed; } + + @Column(name = "id_in_seaweed") + public String getIdInSeaweed() { + return idInSeaweed; + } + + public void setIdInSeaweed(String idInSeaweed) { + this.idInSeaweed = idInSeaweed; + } } diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java index e71226a4a..203b6818b 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java @@ -67,6 +67,13 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i must.add(addTerms("district_name", alias)); must.add(addMatch("cat", "sell")); must.add(addRange("avg_price", minPrice, maxPrice)); + + JSONObject term = new JSONObject(); + JSONObject match = new JSONObject(); + match.put("p_date", 20210501); + term.put("match", match); + must.add(term); + bool.put("must", must); query.put("bool", bool); reqBody.put("query", query); diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/model/Seawater.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/model/Seawater.java index c761ba76a..48005c87b 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/model/Seawater.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/model/Seawater.java @@ -58,6 +58,8 @@ public class Seawater implements SeawaterEntity { private String url; private String root; private Integer avgPrice; + private String purpose; + private String estateType; @Id @Column(name = "_id") @@ -285,4 +287,22 @@ public class Seawater implements SeawaterEntity { public void setAvgPrice(Integer avgPrice) { this.avgPrice = avgPrice; } + + @Column(name = "purpose") + public String getPurpose() { + return purpose; + } + + public void setPurpose(String purpose) { + this.purpose = purpose; + } + + @Column(name = "estate_type") + public String getEstateType() { + return estateType; + } + + public void setEstateType(String estateType) { + this.estateType = estateType; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AuctionController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AuctionController.java index 13e59faa4..ed4a73f2d 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AuctionController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AuctionController.java @@ -23,13 +23,13 @@ import java.util.regex.Pattern; /** * 法拍数据提供接口 */ -@Authority(role = Role.USER) +//@Authority(role = Role.USER) public class AuctionController { private PaiMaiService paiMaiService; private DistrictNewPriceService districtNewPriceService; - private static final String[] SEARCH_FIELDS = new String[]{"marketAvgPrice", "startDate", "curPrice", "imgUrls", "id", "matchCity", "matchRegion", "matchName", "address", "marketTotalPrice", "startPrice", "curPrice", "predictPrice", "depositPrice", "area", "status", "url", "matchEstateType"}; + private static final String[] SEARCH_FIELDS = new String[]{"marketAvgPrice", "startDate", "curPrice", "imgUrls", "id", "matchCity", "matchRegion", "matchName", "address", "marketTotalPrice", "startPrice", "curPrice", "predictPrice", "depositPrice", "area", "status", "url", "matchEstateType","type","count"}; public Object housesAction(AuctionModel model){ RestfulModelAndView mav = new RestfulModelAndView(); @@ -187,6 +187,27 @@ public class AuctionController { return mav; } + + public Object searchPaimaisAction(AuctionModel model) { + RestfulModelAndView mav = new RestfulModelAndView(); + List params = new ArrayList(); + if(StringUtils.isNotBlank(model.getCity())){ + params.add(new QueryParam("matchCity", model.getCity())); + } + if(StringUtils.isNotBlank(model.getDistrict())){ + params.add(new QueryParam("matchRegion", model.getDistrict())); + } + + if(StringUtils.isNotBlank(model.getName())){ + params.add(new QueryParam("matchName", model.getName())); + } + List sortParams = new ArrayList(); + sortParams.add(new SortParam("startDate", false)); + List paimais = paiMaiService.findAll(params, sortParams, model.getLimit(), SEARCH_FIELDS); + mav.setModel(BasicModel.buildOKModel().setResult(paimais)); + return mav; + } + public void setPaiMaiService(PaiMaiService paiMaiService) { this.paiMaiService = paiMaiService; } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/DealPriceController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/DealPriceController.java index 8abece9e2..ab916308a 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/DealPriceController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/DealPriceController.java @@ -15,8 +15,10 @@ import java.util.Map.Entry; import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.fangjia.dal.common.DistrictKey; import com.fangjia.dal.mongo.model.Metro; import com.fangjia.open.model.BasicModel; +import com.fangjia.service.bo.DistrictStatBo; import org.apache.commons.lang.StringUtils; import com.fangjia.core.util.DateUtil; @@ -211,6 +213,42 @@ public class DealPriceController { mv.setModel(resMap); return mv; } + + public Object searchListAction(final DealPriceModel model) { + RestfulModelAndView mv = new RestfulModelAndView(); + mv.setViewType("rest"); + + if (StringUtils.isBlank(model.getId())) { + mv.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "楼盘编号不能为空")); + return mv; + } + District district = districtService.findDistrictById(model.getId()); + DistrictKey districtKey = new DistrictKey(district.getCity(), district.getRegion(), district.getName()); + + if(district == null){ + mv.setModel(BasicModel.buildBadRequestModel("该小区不存在")); + return mv; + } + DistrictStatBo bo = districtStatService.findCurrentDistrictStat(districtKey); + Integer minPrice = null; + Integer maxPrice = null; + if(bo.getDistrictStat()!=null){ + minPrice = (int)(bo.getDistrictStat().getAvgPrice() * 0.8); + maxPrice = (int)(bo.getDistrictStat().getAvgPrice() * 1.2); + } + Date startDate = null; + Date endDate = null; + if(StringUtils.isNotBlank(model.getStartDate())){ + startDate = DateUtil.parseDate(model.getStartDate(),"yyyyMMdd"); + } + if(StringUtils.isNotBlank(model.getEndDate())){ + endDate = DateUtil.parseDate(model.getEndDate(),"yyyyMMdd"); + } + + List dealPrices = dealPriceService.getNearlySameDealPrice(model.getId(), minPrice, maxPrice, startDate, endDate, model.getLimit()); + mv.setModel(BasicModel.buildOKModel().setResult(dealPrices)); + return mv; + } private List findDealPricesByName(String city,String region ,String name){ Calendar calendar = Calendar.getInstance(); diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/DistrictController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/DistrictController.java index 859a04ef0..395cb8a8d 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/DistrictController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/DistrictController.java @@ -7,11 +7,11 @@ import javax.servlet.http.HttpServletRequest; import com.fangjia.core.RegionData; import com.fangjia.core.util.DateUtil; -import com.fangjia.dal.mongo.model.RegionStat; -import com.fangjia.dal.mongo.model.RegionStatRent; +import com.fangjia.dal.mongo.model.*; import com.fangjia.open.model.AvgPriceQueryModel; import com.fangjia.open.model.BasicModel; import com.fangjia.service.datastore.RegionStatService; +import com.fangjia.service.district.DistrictFacilitysService; import com.fangjia.service.district.DistrictStatRentService; import com.fangjia.service.district.RegionStatRentService; import com.fangjia.service.mongo.fangjia.AverageAnalyService; @@ -38,8 +38,6 @@ import com.fangjia.dal.common.DistrictKey; import com.fangjia.dal.common.FangjiaServerAreaContextHolder; import com.fangjia.dal.common.ServerAreaData; import com.fangjia.dal.mongo.fangjia.model.ServerArea; -import com.fangjia.dal.mongo.model.DistrictHouse; -import com.fangjia.dal.mongo.model.DistrictStat; import com.fangjia.dal.mongo.model.bkoff.ExtractDistrict; import com.fangjia.dal.seaweed.model.District; import com.fangjia.dal.sql.model.ApiUser; @@ -93,6 +91,7 @@ public class DistrictController { private RegionBlockService regionBlockService; private RegionStatService regionStatService; private RegionStatRentService regionStatRentService; + private DistrictFacilitysService districtFacilitysService; public static final String[] COLUMNS = { "id", "name", "estateType", "buildType", "licence", "buildArea", "allArea", @@ -444,19 +443,19 @@ public class DistrictController { * * @return */ - public Object showAction(DistrictSearchModel model, @Param(name = "city") String city, @Param(name = "region") String region, - @Param(name = "name") String name, HttpServletRequest request) { - RestfulModelAndView mav = new RestfulModelAndView(); - mav.setViewType("rest"); - if (StringUtils.isBlank(name) || StringUtils.isBlank(city) || StringUtils.isBlank(region)) { - mav.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "小区所在城市,小区名,所在区域必必须输入")); - return mav; - } else { - DistrictBo districtBo = districtService.findByFullMode(new DistrictKey(city, region, name)); - if (null == districtBo) { - mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "小区数据不存在,请检查查询条件")); + public Object showAction(DistrictSearchModel model, @Param(name = "city") String city, @Param(name = "region") String region, + @Param(name = "name") String name, HttpServletRequest request) { + RestfulModelAndView mav = new RestfulModelAndView(); + mav.setViewType("rest"); + if (StringUtils.isBlank(name) || StringUtils.isBlank(city) || StringUtils.isBlank(region)) { + mav.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "小区所在城市,小区名,所在区域必必须输入")); return mav; - } + } else { + DistrictBo districtBo = districtService.findByFullMode(new DistrictKey(city, region, name)); + if (null == districtBo) { + mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "小区数据不存在,请检查查询条件")); + return mav; + } districtBo.setStat(this.districtStatService.findDistrictStat(districtBo.getDistrictKey())); // 去除小区描述中的html标签 districtBo.setDescription(DataUtil.removeSpecialString(districtBo.getDescription())); @@ -1146,6 +1145,25 @@ public class DistrictController { ret.put("RESULT", 200); return mav; } + + public Object facilitysAction(DistrictSearchModel model, @Param(name = "city") String city, @Param(name = "region") String region, + @Param(name = "name") String name, HttpServletRequest request) { + RestfulModelAndView mav = new RestfulModelAndView(); + mav.setViewType("rest"); + if (StringUtils.isBlank(name) || StringUtils.isBlank(city) || StringUtils.isBlank(region)) { + mav.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "小区所在城市,小区名,所在区域必必须输入")); + return mav; + } else { + District districtBo = districtService.findDistrict(new DistrictKey(city, region, name), "district"); + if (null == districtBo) { + mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "小区数据不存在")); + return mav; + } + DistrictFacilitys facilitys = districtFacilitysService.findByDistrict(districtBo); + mav.setModel(BasicModel.buildOKModel().setResult(facilitys)); + } + return mav; + } @@ -1230,4 +1248,7 @@ public class DistrictController { this.regionStatRentService = regionStatRentService; } + public void setDistrictFacilitysService(DistrictFacilitysService districtFacilitysService) { + this.districtFacilitysService = districtFacilitysService; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/EvaluationController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/EvaluationController.java index 67e7b084c..7ed1a83f4 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/EvaluationController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/EvaluationController.java @@ -10,9 +10,14 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import com.fangjia.dal.mongo.model.BsEvaluation; +import com.fangjia.dal.mongo.model.bkoff.DealPrice; import com.fangjia.service.bo.AddrEvaResult; +import com.fangjia.service.bo.DistrictStatBo; import com.fangjia.service.bo.EvaluationBo; import com.fangjia.service.bo.HouseScorer; +import com.fangjia.service.datastore.*; +import com.fangjia.service.district.DistrictStatService; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; @@ -54,11 +59,6 @@ import com.fangjia.open.model.RestfulModelAndView; import com.fangjia.service.api.AutoCompleteService; import com.fangjia.service.api.KnowbaseService; import com.fangjia.service.common.SellSimilar; -import com.fangjia.service.datastore.BusinessEstateService; -import com.fangjia.service.datastore.DistrictService; -import com.fangjia.service.datastore.EvaluationReportService; -import com.fangjia.service.datastore.EvaluationService; -import com.fangjia.service.datastore.HouseService; import com.fangjia.service.sql.ApiUserService; import com.fangjia.service.sql.UserService; import com.fangjia.service.web.AddressService; @@ -94,6 +94,8 @@ public class EvaluationController { private ApiUserService apiUserService; private UserService userService; private AddressService addressService; + private DistrictStatService districtStatService; + private BsEvaluationService bsEvaluationService; private final static String AK = OpenConfig.getConfig().getBaiduAk(); public Object lngLatEvaluateAction(EvaluationModel model) { @@ -1005,6 +1007,44 @@ public class EvaluationController { return mav; } + public Object bsListAction(EvaluationModel model) { + RestfulModelAndView mv = new RestfulModelAndView(); + mv.setViewType("rest"); + + if (StringUtils.isBlank(model.getId())) { + mv.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "楼盘编号不能为空")); + return mv; + } + District district = districtService.findDistrictById(model.getId()); + DistrictKey districtKey = new DistrictKey(district.getCity(), district.getRegion(), district.getName()); + + if(district == null){ + mv.setModel(BasicModel.buildBadRequestModel("该小区不存在")); + return mv; + } + DistrictStatBo bo = districtStatService.findCurrentDistrictStat(districtKey); + Integer minPrice = null; + Integer maxPrice = null; + if(bo.getDistrictStat()!=null){ + minPrice = (int)(bo.getDistrictStat().getAvgPrice() * 0.8); + maxPrice = (int)(bo.getDistrictStat().getAvgPrice() * 1.2); + } + Date startDate = null; + Date endDate = null; + if(StringUtils.isNotBlank(model.getStartDate())){ + startDate = DateUtil.parseDate(model.getStartDate(),"yyyyMMdd"); + } + if(StringUtils.isNotBlank(model.getEndDate())){ + endDate = DateUtil.parseDate(model.getEndDate(),"yyyyMMdd"); + } + + List dealPrices = bsEvaluationService.getNearlySameEvaluation(model.getId(), minPrice, maxPrice, startDate, endDate, model.getLimit()); + mv.setModel(BasicModel.buildOKModel().setResult(dealPrices)); + return mv; + } + + + public void setEvaluationService(EvaluationService evaluationService) { this.evaluationService = evaluationService; } @@ -1017,6 +1057,18 @@ public class EvaluationController { this.knowbaseService = knowbaseService; } + + + public void setDistrictStatService(DistrictStatService districtStatService) { + this.districtStatService = districtStatService; + } + + + + public void setBsEvaluationService(BsEvaluationService bsEvaluationService) { + this.bsEvaluationService = bsEvaluationService; + } + public void setAutoCompleteService(AutoCompleteService autoCompleteService) { this.autoCompleteService = autoCompleteService; } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java index ebb6c9025..bdf38d8b0 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java @@ -11,6 +11,8 @@ import com.fangjia.dal.mongo.model.DistrictFacilitys; import com.fangjia.dal.mongo.model.FangjiaOfficialAreaMap; import com.fangjia.dal.mongo.model.Metro; import com.fangjia.dal.mongo.model.StatsIndex; +import com.fangjia.dal.seawater.dao.SeawaterDao; +import com.fangjia.dal.seawater.model.Seawater; import com.fangjia.dal.seaweed.model.District; import com.fangjia.dal.sql.model.ApiUser; import com.fangjia.datastore.driver.FacetResult; @@ -29,6 +31,7 @@ import com.fangjia.open.ws.query.CollateralSampleQuery; import com.fangjia.open.ws.query.ShopCollateralSampleQuery; import com.fangjia.service.address.AddressParser; import com.fangjia.service.bo.Collateral; +import com.fangjia.service.bo.DistrictStatBo; import com.fangjia.service.bo.Sample; import com.fangjia.service.bo.ShopSample; import com.fangjia.service.datastore.CzbankSampleService; @@ -39,11 +42,13 @@ import com.fangjia.service.datastore.bean.CollateralSampleQueryBean; import com.fangjia.service.datastore.bean.ShopCollateralSampleQueryBean; import com.fangjia.service.district.DistrictFacilitysService; import com.fangjia.service.district.DistrictStatService; +import com.fangjia.service.district.SqmetreRentService; import com.fangjia.service.menu.Item; import com.fangjia.service.menu.MenuCategoryEnum; import com.fangjia.service.menu.Value; import com.fangjia.service.mongo.fangjia.DealPriceService; import com.fangjia.service.mongo.fangjia.MetroService; +import com.fangjia.service.mongo.fangjia.SeawaterService; import com.fangjia.service.mongo.fangjia.StatsIndexService; import com.fangjia.service.open.FangjiaOfficialAreaService; import com.fangjia.service.open.HousePojo; @@ -86,6 +91,9 @@ public class HouseController { private CzbankSampleService czbankSampleService; private DistrictService districtService; private FangjiaOfficialAreaService fangjiaOfficialAreaService; + + private SeawaterService seawaterService; + private SeawaterDao seawaterDao; private List PROPERTY_TYPE_CODE_LIST = Arrays.asList("01", "02"); @@ -399,6 +407,49 @@ public class HouseController { } } + public Object listAction(final HouseSearchModel model) { + RestfulModelAndView mav = new RestfulModelAndView(); + mav.setViewType("rest"); + if (StringUtils.isBlank(model.getId()) && (StringUtils.isBlank(model.getCity()) || StringUtils.isBlank(model.getRegion()) || StringUtils.isBlank(model.getName()))) { + mav.setModel(BasicModel.buildBadRequestModel("楼盘编号或城市,区域,名称为必填项")); + return mav; + } + District district = null; + DistrictKey districtKey = null; + if(StringUtils.isNotBlank(model.getName())){ + districtKey = new DistrictKey(model.getCity(), model.getRegion(), model.getName()); + district = districtService.findDistrict(districtKey, "district"); + }else{ + district = districtService.findDistrictById(model.getId()); + districtKey = new DistrictKey(district.getCity(), district.getRegion(), district.getName()); + } + + if(district == null){ + mav.setModel(BasicModel.buildBadRequestModel("该小区不存在")); + return mav; + } + DistrictStatBo bo = districtStatService.findCurrentDistrictStat(districtKey); + Integer minPrice = null; + Integer maxPrice = null; + if(bo.getDistrictStat()!=null){ + minPrice = (int)(bo.getDistrictStat().getAvgPrice() * 0.8); + maxPrice = (int)(bo.getDistrictStat().getAvgPrice() * 1.2); + } + List names = new ArrayList(); + names.add(district.getName()); + if(district.getAlias() != null){ + for(String alia : district.getAlias()){ + if(StringUtils.isNotBlank(alia) && !names.contains(alia)){ + names.add(alia); + } + } + } + List houses = seawaterDao.findSeawatersFromEs("sell", district.getCity(), district.getRegion(), district.getName(), district.getAlias(), null, model.getStartTime(), model.getEndTime(),null,null,minPrice, maxPrice,null,null,"p_date", false,0, model.getPageSize(),"latest_seawater"); + //List houses1 = seawaterDao.findHouses(district.getCity(), district.getRegion(), names, minPrice, maxPrice, model.getPageSize()); + mav.setModel(BasicModel.buildOKModel().setResult(houses)); + return mav; + } + String getPicUrl(String url, List urls) { if (!urls.contains(url)) { urls.add(url); @@ -832,6 +883,13 @@ public class HouseController { return ret; } + public void setSeawaterService(SeawaterService seawaterService) { + this.seawaterService = seawaterService; + } + + public void setSeawaterDao(SeawaterDao seawaterDao) { + this.seawaterDao = seawaterDao; + } private boolean checkExists(FacetQueryResult houseBoList, HousePojo house) { for (HousePojo bo : houseBoList) { diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index eb9325a57..934ca4cdb 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -665,6 +665,7 @@ public class PropertyController { return mav; } String userId = FangjiaApiUtil.parseUserId(model.getToken()); + District district = null; if (StringUtils.isNotBlank(model.getId())){ if(ObjectId.isValid(model.getId())){ @@ -728,7 +729,7 @@ public class PropertyController { } else { districtInfoDto.setBuildingType(""); } - districtInfoDto.setAlias(district.getAlias()); + districtInfoDto.setAlias(district.getMappingAlias()); districtInfoDto.setAddressAlias(district.getAddrAlias()); districtInfoDto.setCommunity(district.getStreetCommunity()); districtInfoDto.setCommercialArea(district.getBusinessQuarter()); @@ -780,6 +781,7 @@ public class PropertyController { districtInfoDto.setDescription(district.getDescription()); districtInfoDto.setSalesOfficeAdd(district.getSellAddress()); districtInfoDto.setSalesPhoneNumber(district.getMobile()); + districtInfoDto.setTownship(district.getTownship()); if (district.getLoudongCount() != null && district.getLoudongCount() > 0) { districtInfoDto.setBuildingCount(district.getLoudongCount()); } @@ -948,6 +950,12 @@ public class PropertyController { return mav; } + if(model.getMaxFloor() == null && model.getFloor() != null &&StringUtils.isNotBlank(model.getBuildingNumber())){ + Loudong loudong = loudongService.findForEvaluate(district.getId(), model.getBuildingNumber(), model.getUnitNumber()); + if(loudong != null && loudong.getFloors() >= model.getFloor()){ + model.setMaxFloor(loudong.getFloors()); + } + } model.setRegion(district.getRegion()); model.setDistrictName(district.getName()); if (StringUtils.isBlank(model.getPropertyType())) { @@ -1104,7 +1112,7 @@ public class PropertyController { model.setCalDate(calDate); } /* 先到CPDB库中查找小区 */ - District district = matchDistrict(model.getCity(), model.getAddress(), model.getDistrict(), model.getName()); + District district = districtService.findDistrict(new DistrictKey(model.getCity(), model.getDistrict(), model.getName()), "district");//matchDistrict(model.getCity(), model.getAddress(), model.getDistrict(), model.getName()); if(district == null){ //防止地址模糊,名称详细的情况 district = matchDistrict(model.getCity(), model.getName(), model.getDistrict(), model.getAddress()); @@ -5400,6 +5408,36 @@ public class PropertyController { } } + public Object searchHouseLoudongAction(PropertyEvaluationModel model, HttpServletRequest request)throws NumberFormatException, IllegalAccessException { + RestfulModelAndView mav = new RestfulModelAndView(); + mav.setViewType("rest"); + + if (StringUtils.isBlank(model.getId()) || StringUtils.isBlank(model.getHouseNumber()) || StringUtils.isBlank(model.getRoomNumber())){ + mav.setModel(BasicModel.buildBadRequestModel("参数非法:楼盘编号,楼栋号,室为必填项")); + return mav; + } + + List houseLoudongs = houseLoudongService.findAll(model.getId(), model.getHouseNumber(), model.getUnitNumber(), model.getRoomNumber()); + mav.setModel(BasicModel.buildOKModel().setResult(houseLoudongs)); + return mav; + } + + + public Object searchLoudongAction(PropertyEvaluationModel model, HttpServletRequest request)throws NumberFormatException, IllegalAccessException { + RestfulModelAndView mav = new RestfulModelAndView(); + mav.setViewType("rest"); + + if (StringUtils.isBlank(model.getId()) || StringUtils.isBlank(model.getHouseNumber())){ + mav.setModel(BasicModel.buildBadRequestModel("参数非法:楼盘编号,楼栋号为必填项")); + return mav; + } + + + List loudongs = loudongService.findAll(model.getId(), model.getHouseNumber(), model.getUnitNumber()); + mav.setModel(BasicModel.buildOKModel().setResult(loudongs)); + return mav; + } + public void setHouseLoudongService(HouseLoudongService houseLoudongService) { this.houseLoudongService = houseLoudongService; diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/requestController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/requestController.java new file mode 100644 index 000000000..d23a312bb --- /dev/null +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/requestController.java @@ -0,0 +1,36 @@ +package com.fangjia.open.controllers; + +import com.fangjia.core.util.AnalyticsUtil; +import com.fangjia.open.interceptors.Authority; +import com.fangjia.open.interceptors.Role; +import com.fangjia.open.model.BasicModel; +import com.fangjia.open.model.RestfulModelAndView; + +import javax.servlet.http.HttpServletRequest; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Map; + +public class requestController { + + /** + * 获取请求的request和IP信息 + * @param request + * @return + */ + public Object requestAction(HttpServletRequest request) { + RestfulModelAndView mav = new RestfulModelAndView(); + mav.setViewType("rest"); + String ip = AnalyticsUtil.getIpAddress(request); + Map map = new HashMap(); + Enumeration headerNames = request.getHeaderNames(); + while (headerNames.hasMoreElements()) { + String key = (String) headerNames.nextElement(); + String value = request.getHeader(key); + map.put(key, value); + } + map.put("ip", ip); + mav.setModel(BasicModel.buildOKModel().setResult(map)); + return mav; + } +} diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/dto/DistrictInfoDto.java b/projects/fangjia-open/src/main/java/com/fangjia/open/dto/DistrictInfoDto.java index 1ab67359e..d938d3a8b 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/dto/DistrictInfoDto.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/dto/DistrictInfoDto.java @@ -99,6 +99,7 @@ public class DistrictInfoDto { private String salesPhoneNumber; private Integer buildingCount; private Integer finishYear; + private String township; public String getDistrict() { @@ -818,4 +819,12 @@ public class DistrictInfoDto { public void setFinishYear(Integer finishYear) { this.finishYear = finishYear; } + + public String getTownship() { + return township; + } + + public void setTownship(String township) { + this.township = township; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/AuctionModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/AuctionModel.java index a4d47a4e6..a12e09a99 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/AuctionModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/AuctionModel.java @@ -8,6 +8,7 @@ public class AuctionModel extends FangjiaApiModel{ private String city; private String district; private String address; + private String name; private Date minStartTime; private Date maxStartTime; private Integer minStartPrice; @@ -121,4 +122,12 @@ public class AuctionModel extends FangjiaApiModel{ public void setStatus(Integer status) { this.status = status; } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/DealPriceModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/DealPriceModel.java index 0d51d0c46..35620322f 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/DealPriceModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/DealPriceModel.java @@ -1,6 +1,9 @@ package com.fangjia.open.model; +import java.util.Date; + public class DealPriceModel extends FangjiaApiModel{ + private String id; private String city; @@ -34,9 +37,13 @@ public class DealPriceModel extends FangjiaApiModel{ private Double size; private Integer max; + private Integer limit = 10; private String decoration; private boolean convert; + + private String startDate; + private String endDate; public String getCity() { @@ -198,4 +205,36 @@ public class DealPriceModel extends FangjiaApiModel{ public void setConvert(boolean convert) { this.convert = convert; } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getStartDate() { + return startDate; + } + + public void setStartDate(String startDate) { + this.startDate = startDate; + } + + public String getEndDate() { + return endDate; + } + + public void setEndDate(String endDate) { + this.endDate = endDate; + } + + public Integer getLimit() { + return limit; + } + + public void setLimit(Integer limit) { + this.limit = limit; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/EvaluationModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/EvaluationModel.java index d5b8f42ed..71130e7bb 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/EvaluationModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/EvaluationModel.java @@ -246,6 +246,42 @@ public class EvaluationModel extends FangjiaApiModel { private Date buildDate; private String sid;//微信小程序中的sid private String code;//微信小程序中的code + private String id; + private String startDate; + private String endDate; + private Integer limit = 10; + + public Integer getLimit() { + return limit; + } + + public void setLimit(Integer limit) { + this.limit = limit; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getStartDate() { + return startDate; + } + + public void setStartDate(String startDate) { + this.startDate = startDate; + } + + public String getEndDate() { + return endDate; + } + + public void setEndDate(String endDate) { + this.endDate = endDate; + } public String getRegion() { return region; diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/HouseSearchModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/HouseSearchModel.java index d9f4812d8..63a85c4b3 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/HouseSearchModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/HouseSearchModel.java @@ -4,6 +4,7 @@ import com.fangjia.service.datastore.bean.HouseQueryBean; import com.fangjia.service.query.QueryBean; public class HouseSearchModel extends SearchModel { + private String id; private String kind; private String estateType; private String buildingType; @@ -116,7 +117,12 @@ public class HouseSearchModel extends SearchModel { public void setShowDetail(Boolean showDetail) { this.showDetail = showDetail; } - - - + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/PropertyEvaluationModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/PropertyEvaluationModel.java index 5953fc3a3..859658742 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/PropertyEvaluationModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/PropertyEvaluationModel.java @@ -46,7 +46,12 @@ public class PropertyEvaluationModel extends FangjiaApiModel { * 楼号 */ private String unitNumber; + /** + * 新楼号 + */ + private String houseNumber; + private String id; /** * 室号 */ @@ -905,4 +910,20 @@ public class PropertyEvaluationModel extends FangjiaApiModel { public void setAvgPrice(Integer avgPrice) { this.avgPrice = avgPrice; } + + public String getHouseNumber() { + return houseNumber; + } + + public void setHouseNumber(String houseNumber) { + this.houseNumber = houseNumber; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/SearchModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/SearchModel.java index 813f42f6c..8f1311061 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/SearchModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/SearchModel.java @@ -16,6 +16,8 @@ public abstract class SearchModel extends FangjiaApiModel { protected Integer totalPriceEnd; protected Integer avgPriceStart; protected Integer avgPriceEnd; + protected Integer startTime; + protected Integer endTime; protected Integer areaStart; protected Integer areaEnd; protected Integer room; @@ -358,4 +360,20 @@ public abstract class SearchModel extends FangjiaApiModel { public void setTotalPrice(Integer totalPrice) { this.totalPrice = totalPrice; } + + public Integer getStartTime() { + return startTime; + } + + public void setStartTime(Integer startTime) { + this.startTime = startTime; + } + + public Integer getEndTime() { + return endTime; + } + + public void setEndTime(Integer endTime) { + this.endTime = endTime; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy index 42c580735..6ef05438e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy @@ -68,20 +68,30 @@ class BigdataBuildingBatch extends BaseBatch{ def q = new BasicDBObject() q.put("city", city) q.put("status", 1) +// q.put("name","辽源一村") def dbcursor = LOU_DONG_STAT.find(q) def map = [:] + def set = new HashSet() for(def db : dbcursor){ + def list = map.get(db.region+"##"+db.name) if(!list){ list = [] map.put(db.region+"##"+db.name, list) } - list << db + if(db.house_num && db.house_num!="--"){ + def key = db.region+"##"+db.name +"##"+db.house_num + if(!set.contains(key)){ + set << key + list << db + } + } } return map } def createCityDatas(cityCodeDatas, distCodeMap, distMap, buildingMap){ def rets = [] + def keys = [] distMap.each{k,dist-> def buildings = buildingMap.get(k) if(buildings){ @@ -89,53 +99,57 @@ class BigdataBuildingBatch extends BaseBatch{ def ret = [] def areaData = distCodeMap.get(obj.region) if(areaData){ - ret << cityCodeDatas.get(obj.city) - ret << obj._id.toString() - ret << obj.house_num - ret << "" - ret << "" - ret << areaData.district_code - ret << dist._id.toString() - if(obj.build_year){ - ret << obj.build_year - }else{ + def key = dist._id.toString() +"##"+obj.house_num + if(!keys.contains(key)){ + keys << key + ret << cityCodeDatas.get(obj.city) + ret << obj._id.toString() + ret << obj.house_num + ret << obj.house_num ret << "" - } - def arrs = CcbCodeUtils.parseBuildType(dist.build_type) - if(arrs){ - ret << arrs[0].trim() - }else{ - ret << '' - } + ret << areaData.district_code + ret << dist._id.toString() + if(obj.build_year){ + ret << obj.build_year + }else{ + ret << "" + } + def arrs = CcbCodeUtils.parseBuildType(dist.build_type) + if(arrs){ + ret << arrs[0].trim() + }else{ + ret << '' + } + + ret << CcbCodeUtils.getBuildStructureCode(dist.build_structure) + if(obj.floors){ + ret << obj.floors + }else{ + ret << "" + } - ret << CcbCodeUtils.getBuildStructureCode(dist.build_structure) - if(obj.floors){ - ret << obj.floors - }else{ ret << "" - } - - ret << "" - ret << "" - ret << "" + ret << "" + ret << "" // if(obj.elevators && obj.elevators>=1){ // ret << "1" // }else{ // ret << "0" // } - ret << "" + ret << "" // if(obj.elevators && obj.elevators>=1) { // ret << obj.elevators//电梯数 // }else{ // ret << "" // } - ret << "" - ret << "" - ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss") - ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss") - ret << "FJW" - ret << "1" - rets << ret + ret << "" + ret << "" + ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss") + ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + ret << "1" + rets << ret + } } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 8a8b4a7dc..98fbc2d09 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -367,7 +367,7 @@ class CcbCommonUtils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name","特房锦绣一方") +// q.put("name","辽源一村") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy index 3fffa870c..74474aff6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy @@ -57,6 +57,12 @@ class CcbFtpUtils { ftpUpload(dataCodeFileName)//上传加密码文件到FTP } + static def listFileNames(month){ + //上传到ftp + sftpChannel = reConnect(); + Vector files = sftp.listFiles(ftpPath+ month+"/", sftpChannel); + return files + } static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index f75d28656..d7840e5bb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -25,8 +25,8 @@ class DealPriceDataProcess { def task = new DealPriceDataProcess() // task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据 // task.addLianJiaPrice(); -// task.deleteThirdPartyData() - task.delDealByPe() + task.deleteThirdPartyData() +// task.delDealByPe() } def delDealByPe(){ @@ -370,26 +370,26 @@ class DealPriceDataProcess { def deleteThirdPartyData(){ - def citys = ["东莞"] //,"杭州""广州", + def citys = CcbCommonUtils.citys def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def publishTime = '2021-10-01' - def month = 202110 + def month = 202203 def rows = [] for(def city : citys) { - def rets = findPeGteThirdParty(city, 202111) + def rets = findPeGteThirdParty(city, 202204) def total = 0 for(def m : rets){ def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}" - sqlInstance.executeUpdate(sql) + sqlInstance236.executeUpdate(sql) } - println city +" "+total + println city +" "+rets.size() } } def findPeGteThirdParty(cityName, yearmonth){ - def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>20" + def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>50" def rets = [] - sqlInstance2.eachRow(sql,{r-> + sqlInstance236.eachRow(sql,{r-> def map = [:] map.commid = r.commid map.price = r.target_avg_price diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 9d06d7cef..060914122 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -38,8 +38,8 @@ class ExportOracleData { exportCommunityBuileType() exportThirdPartyPrice2(lastMonth) - exportBigdataForsale("2022-02-19") - exportBigdataDeal("2022-02-01") + exportBigdataForsale("2022-03-19") + exportBigdataDeal("2022-03-01") //计算结果相关 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy index edc128cf1..7258934fa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy @@ -3,8 +3,10 @@ package com.fangjia.script.ccbevaluation import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.util.DateUtils +import com.jcraft.jsch.ChannelSftp import groovy.sql.Sql import com.fangjia.script.ccbevaluation.ronghe.SSHHelper +import org.apache.commons.codec.digest.DigestUtils class ExportOracleEvaluationData { def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", @@ -33,6 +35,8 @@ class ExportOracleEvaluationData { exportCommCaseDetailGj(curMonth) def curDate = curMonth.toString()+"28" +// 生成MD5文件 + exportMd5() SSHHelper jschCommand = new SSHHelper(); System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}")); // ok文件 @@ -413,4 +417,23 @@ class ExportOracleEvaluationData { CcbFtpUtils.ftpUpload(null, dataCodeFileName, curDate) } + def exportMd5(){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + Vector files = CcbFtpUtils.listFileNames(curDate) + Iterator fileVec = files.iterator() + def fileRows = [] + while(fileVec.hasNext()){ + ChannelSftp.LsEntry file = fileVec.next() + if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){ + fileRows << file.getFilename() +":"+ DigestUtils.md5Hex(file.getFilename()) + } + } + def fileName = "/ccb/md5.txt" + if(CommonDatautils.isWindow()){ + fileName = "D:\\房价网\\md5.txt" + } + FileUtils.createTxtFile(fileRows, fileName, "utf-8", false); + CcbFtpUtils.ftpUpload(null, fileName, curDate) + println "md5.txt 上传Ftp完成......" + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 421a0e4f7..e262bd7c5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202202//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202112 - def endDate = 202202 + def sDate= 202203//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202201 + def endDate = 202203 def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index afbaed5ee..3ed13ebfc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -75,13 +75,13 @@ class TradeStatLog { def run(){ def citys = CcbCommonUtils.citys; def rets = [] - rets << ["城市","12月总量","1月总量","2月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] + rets << ["城市","1月总量","2月总量","3月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"] //def startDate = "2021-05-01" - def startDate = "2021-12-01" - def date = "2022-01-01" - def endDate = "2022-02-01" - def newDate = "2022-03-01" + def startDate = "2022-01-01" + def date = "2022-02-01" + def endDate = "2022-03-01" + def newDate = "2022-04-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) @@ -131,7 +131,7 @@ class TradeStatLog { } - ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-02成交统计.xlsx", rets, "数据") + ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-03成交统计.xlsx", rets, "数据") println "表格生成完成" } @@ -481,8 +481,8 @@ class TradeStatLog { def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"] def rets = [] rets << headers - def startDate = "2022-02-01" - def endDate = "2022-03-01" + def startDate = "2022-03-01" + def endDate = "2022-04-01" for(def city : citys){ def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy index 5df3fb0fd..2db54e10a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy @@ -2,25 +2,243 @@ package com.fangjia.script.ccbevaluation.data import com.fangjia.core.RegionData import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.core.util.DateUtil import com.fangjia.core.util.GeoUtils import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.OpenApiUtils import com.fangjia.script.linxsh.util.AddressUtil import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.GeoHelper import com.mongodb.BasicDBObject +import java.util.concurrent.CopyOnWriteArrayList + class CcbDistrictToDB { static main(args) { def task = new CcbDistrictToDB(); // task.run() - task.run1() +// task.run1() +// task.meituanRun() +// task.meituanSqrentRun() +// task.updateDistrict() + task.updateDongguangRegion() } + + def updateDongguangRegion(){ + def cDate = DateUtils.parseFromTightByPattern("2022-04-27","yyyy-MM-dd") + def dist = CommonDatautils.findAllDistrictByCity("东莞", "district", ["city","region","name","c_date","estate_type2"]) + for(def d : dist){ + def t = d.c_date + if(t.getTime()>cDate.getTime()){ + + } + } + } + + def updateDistrict(){ + def citys = ["惠州","东莞","金华","苏州","桂林","潮州","廊坊","临沧","通辽","雅安"] + def cDate = DateUtils.parseFromTightByPattern("2022-04-27","yyyy-MM-dd") + def index = 0 + for(def city : citys){ + def dist = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","c_date","estate_type2"]) + for(def d : dist){ + def t = d.c_date + if(t.getTime()>cDate.getTime()){ + if(d.name.contains("(")){ + def delstr = d.name.substring(d.name.indexOf("("), d.name.indexOf(")")+1) + def newName = d.name.replace(delstr,"") + index++ + println index+" "+d.city+" "+d.name+" "+newName + def obj = CommonDatautils.findDistrict(d.city, d.region, newName, "district") + if(!obj){ + println "update" + def udata = new BasicDBObject() + udata.put("name", newName) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + + d.name = newName + addPrice(d) + + def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1)); + CommonUtils.recalculationSqRent(d.city, d.region, newName, d.estate_type2, lastMonthDay,false,0) + }else{ + println "delete" + DbUtils.seaweed.remove(new BasicDBObject("_id",d._id)) + } + } + } + } + } + } + + def meituanSqrentRun(){ + ExcelUtils.modify("D:\\数据\\美团\\美团需入库小区合并后.xlsx", { d -> + def city = d['F'] + def region = d['G'] + def newRegion = RegionData.formatRegion(city, region) + if(newRegion) { + def name = d['H'] + name = nameFileter(name) + if(name){ + def estateType2 = "普通住宅" + if(name.contains("别墅")){ + estateType2 = "别墅" + }else if(name.contains("商住")){ + estateType2 = "商住" + }else{ + def type = d['I'] + if(type){ + if(type.contains("别墅")){ + estateType2 = "别墅" + }else if(type.contains("商住")){ + estateType2 = "商住" + } + } + } + + def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1)); + CommonUtils.recalculationSqRent(city, region, name, estateType2, lastMonthDay,false,0) + } + } + },0); + } + + def meituanRun(){ + def index = 0//美团需入库小区合并后.xlsx + def keys = new CopyOnWriteArrayList() + + ExcelUtils.multiThreadingModify("D:\\数据\\美团\\美团需入库小区合并后.xlsx", { d -> + def city = d['F'] + def region = d['G'] + def newRegion = RegionData.formatRegion(city, region) + if(newRegion){ + def name = d['H'] + name = nameFileter(name) + + def formatAddress = null + if(d['E']){ + formatAddress = AddressUtil.formatAddress(city, d['E']) + } + + if(!formatAddress || (formatAddress && !formatAddress.contains("号") && !formatAddress.contains("交叉"))){ + def rets = OpenApiUtils.getGaoDeSearchLocation(city, region, name) + def map = findStandardObject2(city,region, name, d['E'], rets); + if(map && map.address){ + formatAddress = AddressUtil.formatAddress(city, map.address) + if(!formatAddress){ + if(map.address.contains("交叉口")){ + formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉口")+3) + }else if(map.address.contains("交叉")){ + formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉")+2) + } + } + } + } + + + Long glat = null + Long glng = null + Long blat = null + Long blng = null + if(d['K']){ + glat = d['L'].toString().toDouble() * 10000000000L + glng = d['K'].toString().toDouble() * 10000000000L + def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3) + blng = baidu.x * 10000000000L + blat = baidu.y * 10000000000L + } + + def estateType2 = "普通住宅" + if(name.contains("别墅")){ + estateType2 = "别墅" + }else if(name.contains("商住")){ + estateType2 = "商住" + }else{ + def type = d['I'] + if(type){ + if(type.contains("别墅")){ + estateType2 = "别墅" + }else if(type.contains("商住")){ + estateType2 = "商住" + } + } + } + + def key = city+"##"+newRegion+"##"+name + if(name && !keys.contains(key)){ + def dist = CommonDatautils.findDistrict(city, newRegion, name, "district") + if(!dist){ + keys << key + def data = new BasicDBObject(); + data.put("name_cat",0) + data.put("status",0) + data.put("cat","district") + data.put("c_date", new Date()) + data.put("m_date", new Date()) + data.put("city", city) + data.put("region",newRegion) + data.put("name", name) + data.put("estate_type2",estateType2) + data.put("address", formatAddress) + if(glat){ + data.put("lat",glat) + data.put("lng",glng) + data.put("lat2",blat) + data.put("lng2",blng) + } + def markMap = [:] + markMap.put("data_source", "机器入库") + data.put("mark_map",markMap) + index++ + DbUtils.seaweed.save(data); + + + println index+" "+data + + def stop = false + while (!stop){ + if(data._id.toString()){ + addPrice(data) + stop = true; + } + } + + } + } + }else{ + println city +" "+region +" 区域无法标准化" + } + },0); + + + + } + + + def addPrice(dist){ + def startDate = DateUtils.parseFromTightYYYYMMDD("2022-03-01") + def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-25") + def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate) + + for(def weekend : weekends){ + CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0) + } + } + + def run1(){ def index = 0 - ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\新增小区数据_1236.xlsx", { d -> + + + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\新增小区数据_1236.xlsx", { d -> def city = d['A'] def region = d['B'] def newRegion = RegionData.formatRegion(city, region) @@ -71,6 +289,7 @@ class CcbDistrictToDB { if(name){ def dist = CommonDatautils.findDistrict(d['A'], newRegion, name, "district") if(!dist){ + def data = new BasicDBObject(); data.put("name_cat",0) data.put("status",0) @@ -221,4 +440,77 @@ class CcbDistrictToDB { name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区") return name; } + + + def findStandardObject2(def city, def region, def name, def address, def rets) { + if (rets) { + def bestObj = [:] + if (rets.size() >= 1) { + for (def mapObj : rets) { + def score = 0 + if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){ + score = 0.2 + } + if(mapObj.name && name){ + def newName = mapObj.name; + def names = [] + if(mapObj.name && mapObj.name.indexOf("(")>-1){ + def name1 = mapObj.name.substring(0, mapObj.name.indexOf("(")) + names << name1 + def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")")) + names << name2 + }else{ + names << newName + } + def maxNameScore = 0 + for(def n : names){ + double score1 = CosineSimilarAlgorithm.getSimilarity(n, name); + if(score1 && score1>0){ + def nameScore = (score1 * 4)/10 + if(nameScore > maxNameScore){ + maxNameScore = nameScore + } + } + } + + if(maxNameScore){ + try{ + maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore) + }catch(Exception e){ + maxNameScore = 0 + } + + } + score = maxNameScore + score; + } + + if(mapObj.address && address){ + double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address); + if(addressScore && addressScore>0){ + try{ + addressScore = DataHandleUtils.getDoubleRound(2, addressScore) + }catch(Exception e){ + addressScore = 0 + } + score = addressScore*4/10 + score + } + } + + if(!bestObj){ + mapObj.score = score + bestObj = mapObj + }else{ + if(score > bestObj.score){ + mapObj.score = score + bestObj = mapObj + } + } + + } + } + return bestObj + } + return null + + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/BlankEvaluationBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/BlankEvaluationBatch.groovy index 88f5c3bdf..e45fc62c6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/BlankEvaluationBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/BlankEvaluationBatch.groovy @@ -16,7 +16,9 @@ import com.mongodb.DBObject import groovy.sql.Sql import org.apache.lucene.queryparser.classic.QueryParser - +/** + * 重庆估值测试--2小时 + */ class EvaluationBatch { public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; @@ -27,8 +29,8 @@ class EvaluationBatch { static main(args) { def task = new EvaluationBatch() // task.run() - task.run1(); -// task.run2() +// task.run1(); + task.run2() } def run2(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/BlankMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/BlankMatch.groovy index 75c731a7f..77b853dad 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/BlankMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/BlankMatch.groovy @@ -8,42 +8,17 @@ import com.fangjia.script.util.OpenApiClient class BlankMatch { static main(args) { def task = new BlankMatch() -// task.run() + task.run() // task.run1() // task.run2() - task.run3() - } - - def run3(){ - ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row -> - if(!row['C']){ - def dist = CommonDatautils.findDistrict("重庆", row['E'], row['F'], "district") - if(dist){ - row['C'] = dist._id.toString() - } - } - },0) - } - def run2(){ - def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配不上(合并).xlsx",0,1) - ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row -> - if(!row['C']){ - for(def d : datas){ - if(d['B'] == row['B']){ - row['C'] = d['C'] - row['D'] = d['D'] - row['E'] = d['E'] - row['F'] = d['J'] - break; - } - } - } - },0) +// task.run3() } def run() { //rMap = SeaweedDataMatchProcess.doMatch(city, region, d['G'], null); - ExcelUtils.modify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row -> + //def path = "D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx"; + def path = "D:\\数据\\重庆农商行\\2022-04-02\\测试.xlsx" + ExcelUtils.multiThreadingModify(path, { row -> if(!row['C']){ def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null) if(jsonObject != null && jsonObject.precise!=-1) { @@ -54,8 +29,6 @@ class BlankMatch { row['G'] = jsonObject.get("address") } } - - }); } @@ -81,4 +54,31 @@ class BlankMatch { // } }); } + + def run3(){ + ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row -> + if(!row['C']){ + def dist = CommonDatautils.findDistrict("重庆", row['E'], row['F'], "district") + if(dist){ + row['C'] = dist._id.toString() + } + } + },0) + } + def run2(){ + def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配不上(合并).xlsx",0,1) + ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row -> + if(!row['C']){ + for(def d : datas){ + if(d['B'] == row['B']){ + row['C'] = d['C'] + row['D'] = d['D'] + row['E'] = d['E'] + row['F'] = d['J'] + break; + } + } + } + },0) + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy new file mode 100644 index 000000000..cd0df4d4c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy @@ -0,0 +1,254 @@ +package com.fangjia.script.chongqing + +import com.fangjia.core.RegionData +import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.gj.paimai.DataUtil +import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import groovy.sql.Sql + +import java.util.concurrent.CopyOnWriteArrayList + +class MatchEvaluateRegionTest { + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx" + def total = 1500 + + static main(args) { + def task = new MatchEvaluateRegionTest() + task.run()//小区匹配 + task.run1()//地图匹配 +// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除 + task.addPrice() + task.addArroundPrice() + task.addAuthor() + } + def run(){ + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['F']){ + def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], "property", false, null, null) + if(jsonObject != null && jsonObject.precise!=-1) { + row['F'] = jsonObject.get("id") + row['G'] = jsonObject.get("city") + row['H'] = jsonObject.get("district") + row['I'] = jsonObject.get("name") + row['J'] = jsonObject.get("address") + row['K'] = jsonObject.get("propertyType") + row['L'] = jsonObject.get("map") +// row['K'] = jsonObject.get("lng") +// row['L'] = jsonObject.get("lat") + } + } + }); + } + + def run1(){ + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['F']){ + def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], null, true, null, null) + if(jsonObject != null && jsonObject.precise!=-1) { + row['F'] = jsonObject.get("id") + row['G'] = jsonObject.get("city") + row['H'] = jsonObject.get("district") + row['I'] = jsonObject.get("name") + row['J'] = jsonObject.get("address") + row['K'] = jsonObject.get("propertyType") + row['L'] = jsonObject.get("map") +// row['K'] = jsonObject.get("lng") +// row['L'] = jsonObject.get("lat") + } + } + }); + } + + def addPrice(){ + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['L']){ + def priceObj = CommonDatautils.findDistrictPrice(row['G'], row['H'], row['I']) + if(priceObj){ + row['M'] = priceObj.avg_price + row['N']= (int)(priceObj.avg_price * row['E']) + } + + def avg_price = findAvgPrice(row['M']) + if(row['M'] && avg_price){ + def weekendPrice = row['M'].toString().toDouble() + def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price) + row['O'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15) + } + } + }); + + } + + def addArroundPrice(){ + ExcelUtils.multiThreadingModify(path, { row -> + if(row['A'] && !row['K']) { + def arroundObj = FangjiaClientUtils.arroundPrice(row['A'], row['B']) + if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) { + def priceObj = arroundObj.result + row['M'] = priceObj.unitPrice + row['N'] = (int) (priceObj.unitPrice * row['C']) + row['Q'] = "周边价格" + } + } + }); + } + + def addAuthor(){ + def authors = ["丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"] + def total = 747 + def personTotal = authors.size() + def avg = Math.ceil(total/personTotal) + + def index = 0 + def count = 0; + ExcelUtils.modify(path, { row -> + if(index < avg){ + row['P'] = authors[count] + } + index++ + if(index >= avg){ + count++ + index = 0 + } + }); + } + + + def addDistrict(){ + def path = "D:\\数据\\重庆农商行\\2022-05-06\\需新增小区.xlsx" + def keys = new CopyOnWriteArrayList() + def index = 0 + ExcelUtils.modify(path, { d-> + def flag = d['M'] + if(!flag){ + def city = d['E'] + def region = d['F'] + def newRegion = RegionData.formatRegion(city, region) + if(newRegion){ + def name = d['G'] + name = nameFileter(name) + + def formatAddress = null + def address = d['H'] + if(address){ + formatAddress = AddressUtil.formatAddress(city, address) + if(!formatAddress){ + if(address.contains("交叉口")){ + formatAddress = address.substring(0, address.toString().indexOf("交叉口")+3) + }else if(address.contains("交叉")){ + formatAddress = address.substring(0, address.toString().indexOf("交叉")+2) + } + } + } + + Long glat = null + Long glng = null + Long blat = null + Long blng = null + if(d['K']){ + glat = d['L'].toString().toDouble() * 10000000000L + glng = d['K'].toString().toDouble() * 10000000000L + def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3) + blng = baidu.x * 10000000000L + blat = baidu.y * 10000000000L + } + + def estateType2 = "普通住宅" + if(name.contains("别墅")){ + estateType2 = "别墅" + }else if(name.contains("商住")){ + estateType2 = "商住" + }else{ + def type = d['I'] + if(type){ + if(type.contains("别墅")){ + estateType2 = "别墅" + }else if(type.contains("商住")){ + estateType2 = "商住" + } + } + } + + def key = city+"##"+newRegion+"##"+name + if(name && !keys.contains(key)){ + def dist = CommonDatautils.findDistrict(city, newRegion, name, "district") + if(!dist){ + keys << key + def data = new BasicDBObject(); + data.put("name_cat",0) + data.put("status",0) + data.put("cat","district") + data.put("c_date", new Date()) + data.put("m_date", new Date()) + data.put("city", city) + data.put("region",newRegion) + data.put("name", name) + data.put("estate_type2",estateType2) + data.put("address", formatAddress) + if(glat){ + data.put("lat",glat) + data.put("lng",glng) + data.put("lat2",blat) + data.put("lng2",blng) + } + def markMap = [:] + markMap.put("data_source", "机器入库") + data.put("mark_map",markMap) + index++ + DbUtils.seaweed.save(data); + println index+" "+data + addPrice(data) + } + } + }else{ + println city +" "+region +" 区域无法标准化" + } + } + },0) + } + + def addPrice(dist){ + def startDate = DateUtils.parseFromTightYYYYMMDD("2022-03-01") + def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-25") + def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate) + + for(def weekend : weekends){ + CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0) + } + } + + + def nameFileter(def name){ + name = name.replace("·","").replace(".","").replace("|","") + name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期") + name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区") + if(name.contains("(")){ + def delstr = name.substring(name.indexOf("("), name.indexOf(")")+1) + name = name.replace(delstr,"") + } + return name; + } + + def findAvgPrice(def commid){ + def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2"; + def avg_price = null + sqlInstance.eachRow(sql, { r -> + avg_price = r.target_avg_price + }); + return avg_price + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateTest.groovy new file mode 100644 index 000000000..936b3b582 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateTest.groovy @@ -0,0 +1,280 @@ +package com.fangjia.script.chongqing + +import com.fangjia.core.RegionData +import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.gj.paimai.DataUtil +import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import groovy.sql.Sql + +import java.util.concurrent.CopyOnWriteArrayList +import com.fangjia.script.gj.district.FangjiaClientUtils + +class MatchEvaluateTest { + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + def path = "D:\\数据\\重庆农商行\\2022-05-06\\模拟_测试数据.xlsx" + def total = 1500 + + static main(args) { + def task = new MatchEvaluateTest() +// task.run()//小区匹配 +// task.run1()//地图匹配 +//// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除 +// task.addPrice() +// task.addArroundPrice() +// task.addAuthor() + + task.stat() + } + + def addArroundPrice(){ + ExcelUtils.multiThreadingModify(path, { row -> + if(row['A'] && !row['K']) { + def arroundObj = FangjiaClientUtils.arroundPrice(row['A'], row['B']) + if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) { + def priceObj = arroundObj.result + row['K'] = priceObj.unitPrice + row['L'] = (int) (priceObj.unitPrice * row['C']) + row['O'] = "周边价格" + } + } + }); + } + + + def run(){ +// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx" + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['D'] && row['B']){ + def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null) + if(jsonObject != null && jsonObject.precise!=-1) { + row['D'] = jsonObject.get("id") + row['E'] = jsonObject.get("city") + row['F'] = jsonObject.get("district") + row['G'] = jsonObject.get("name") + row['H'] = jsonObject.get("address") + row['I'] = jsonObject.get("propertyType") + row['J'] = jsonObject.get("map") +// row['K'] = jsonObject.get("lng") +// row['L'] = jsonObject.get("lat") + } + } + }); + } + + def run1(){ +// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx" + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['D'] && row['B']){ + def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], null, true, null, null) + if(jsonObject != null && jsonObject.precise!=-1) { + row['D'] = jsonObject.get("id") + row['E'] = jsonObject.get("city") + row['F'] = jsonObject.get("district") + row['G'] = jsonObject.get("name") + row['H'] = jsonObject.get("address") + row['I'] = jsonObject.get("propertyType") + row['J'] = jsonObject.get("map") +// row['K'] = jsonObject.get("lng") +// row['L'] = jsonObject.get("lat") + } + } + }); + } + + def addPrice(){ +// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx" + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['J'] && row['E']){ + def priceObj = CommonDatautils.findDistrictPrice(row['E'], row['F'], row['G']) + if(priceObj){ + row['K'] = priceObj.avg_price + row['L']= (int)(priceObj.avg_price * row['C']) + } + + + def avg_price = findAvgPrice(row['D']) + if(row['K'] && avg_price){ + def weekendPrice = row['K'].toString().toDouble() + def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price) + row['M'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15) + } + } + }); + + } + + def addAuthor(){ +// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx" + def authors = ["丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"] + def personTotal = authors.size() + def avg = Math.ceil(total/personTotal) + + def index = 0 + def count = 0; + ExcelUtils.modify(path, { row -> + if(row['B']){ + if(index < avg){ + row['N'] = authors[count] + } + index++ + if(index >= avg){ + count++ + index = 0 + } + } + }); + } + + + def addDistrict(){ + def path = "D:\\数据\\重庆农商行\\2022-05-06\\需新增小区.xlsx" + def keys = new CopyOnWriteArrayList() + def index = 0 + ExcelUtils.modify(path, { d-> + def flag = d['M'] + if(!flag){ + def city = d['E'] + def region = d['F'] + def newRegion = RegionData.formatRegion(city, region) + if(newRegion){ + def name = d['G'] + name = nameFileter(name) + + def formatAddress = null + def address = d['H'] + if(address){ + formatAddress = AddressUtil.formatAddress(city, address) + if(!formatAddress){ + if(address.contains("交叉口")){ + formatAddress = address.substring(0, address.toString().indexOf("交叉口")+3) + }else if(address.contains("交叉")){ + formatAddress = address.substring(0, address.toString().indexOf("交叉")+2) + } + } + } + + Long glat = null + Long glng = null + Long blat = null + Long blng = null + if(d['K']){ + glat = d['L'].toString().toDouble() * 10000000000L + glng = d['K'].toString().toDouble() * 10000000000L + def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3) + blng = baidu.x * 10000000000L + blat = baidu.y * 10000000000L + } + + def estateType2 = "普通住宅" + if(name.contains("别墅")){ + estateType2 = "别墅" + }else if(name.contains("商住")){ + estateType2 = "商住" + }else{ + def type = d['I'] + if(type){ + if(type.contains("别墅")){ + estateType2 = "别墅" + }else if(type.contains("商住")){ + estateType2 = "商住" + } + } + } + + def key = city+"##"+newRegion+"##"+name + if(name && !keys.contains(key)){ + def dist = CommonDatautils.findDistrict(city, newRegion, name, "district") + if(!dist){ + keys << key + def data = new BasicDBObject(); + data.put("name_cat",0) + data.put("status",0) + data.put("cat","district") + data.put("c_date", new Date()) + data.put("m_date", new Date()) + data.put("city", city) + data.put("region",newRegion) + data.put("name", name) + data.put("estate_type2",estateType2) + data.put("address", formatAddress) + if(glat){ + data.put("lat",glat) + data.put("lng",glng) + data.put("lat2",blat) + data.put("lng2",blng) + } + def markMap = [:] + markMap.put("data_source", "机器入库") + data.put("mark_map",markMap) + index++ + DbUtils.seaweed.save(data); + println index+" "+data + addPrice(data) + } + } + }else{ + println city +" "+region +" 区域无法标准化" + } + } + },0) + } + + def addPrice(dist){ + def startDate = DateUtils.parseFromTightYYYYMMDD("2022-03-01") + def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-25") + def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate) + + for(def weekend : weekends){ + CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0) + } + } + + + def nameFileter(def name){ + name = name.replace("·","").replace(".","").replace("|","") + name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期") + name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区") + if(name.contains("(")){ + def delstr = name.substring(name.indexOf("("), name.indexOf(")")+1) + name = name.replace(delstr,"") + } + return name; + } + + def findAvgPrice(def commid){ + def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2"; + def avg_price = null + sqlInstance.eachRow(sql, { r -> + avg_price = r.target_avg_price + }); + return avg_price + } + + def stat(){ + def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\2022-05-06\\模拟_测试数据_合并后 (1).xlsx",0,1) + ExcelUtils.multiThreadingModify(path, { d -> + if(d['B']){ + for(def row : datas){ + if(d['B']==row['B']){ + if(row['K']){ + d['P'] = row['K'] + break + } + } + } + } + }); + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocTest.groovy new file mode 100644 index 000000000..804d5891a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocTest.groovy @@ -0,0 +1,510 @@ +package com.fangjia.script.chongqing + +import com.fangjia.core.util.SourceNameUtil +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient + +import java.util.concurrent.CopyOnWriteArrayList + +class PocTest { + + static main(args) { + def task = new PocTest(); + task.run(); + } + + def run(){ +// match("D:\\数据\\重庆农商行\\poc\\test.xlsx") +// addPrice("D:\\数据\\重庆农商行\\poc\\test.xlsx") +// standard("D:\\数据\\重庆农商行\\poc\\test.xlsx") +// addLoudongNum("D:\\数据\\重庆农商行\\poc\\test.xlsx") +// addDetail("D:\\数据\\重庆农商行\\poc\\test.xlsx", "D:\\数据\\重庆农商行\\poc\\楼盘详情.xlsx") + +// createLoudong("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\底层数据-楼栋-户.xlsx") +// createPaimai("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\法拍案例.xlsx") +// createHouse("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\案例.xlsx") +// createDealPrice("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\案例.xlsx") + createEvaluation("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\案例.xlsx") + } + + + def match(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['P'] && row['C']){ + def jsonObject = OpenApiClient.addressMatch(row['A'], row['C'], "property", false, row['B'], null) + if(jsonObject != null && jsonObject.precise!=-1) { + row['P'] = jsonObject.get("name") + row['Q'] = jsonObject.get("id") + } + } + }) + } + + def addPrice(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(row['Q']){ + def dist = CommonDatautils.findDistrictById(row['Q']) + if(dist){ + def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name) + if(priceObj){ + row['E'] = priceObj.avg_price + } + } + } + }) + } + + def standard(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(row['C']){ + def jsonObject = OpenApiClient.standard2Action(row['A'], row['C']) + if(jsonObject){ + println jsonObject + row['F'] = jsonObject.get("province") + row['G'] = jsonObject.get("city") + row['H'] = jsonObject.get("district") + row['I'] = jsonObject.get("township") + row['J'] = jsonObject.get("street") + row['K'] = jsonObject.get("streetNumber") + row['L'] = jsonObject.get("buildingNumber") + row['M'] = jsonObject.get("unitNumber") + row['N'] = jsonObject.get("floor") + row['O'] = jsonObject.get("roomNumber") + } + } + + }) + } + + def addLoudongNum(path){ + + ExcelUtils.multiThreadingModify(path, { row -> + if (row['Q'] && row['L']) { + def jsonObject = OpenApiClient.searchLoudong(row['Q'], row['L'], row['M']) + if (jsonObject && jsonObject.result) { + if(jsonObject.result.size()>=1){ + def obj = jsonObject.result[0] + row['S'] = obj.id + row['R'] = obj.houseNumber + } + } + + if(row['O']){ + def jsonObject1 = OpenApiClient.searchHouseLoudong(row['Q'], row['L'], row['M'], row['O']) + if (jsonObject1 && jsonObject1.result) { + if(jsonObject.result.size()>=1){ + def obj = jsonObject.result[0] + row['U'] = obj.id + row['T'] = obj.roomNum + } + } + } + } + }) + } + + def createLoudong(path, output){ + def rets = new CopyOnWriteArrayList() + def datas = ExcelUtils.read(path, 0, 1) + rets << ["输入城市","输入区域","输入地址","面积","楼栋号","楼栋ID","物业类型","主朝向","地上层数","地下层数","竣工日期","梯","户","总户数","景观","户号","户ID","主朝向","景观","户型结构"] + for(def row : datas){ + def data = [] + data << row['A'] + data << row['B'] + data << row['C'] + data << row['D'] + if (row['Q'] && row['L']) { + def jsonObject = OpenApiClient.searchLoudong(row['Q'], row['L'], row['M']) + if (jsonObject && jsonObject.result && jsonObject.result.size()>=1) { + def obj = jsonObject.result[0] + data << obj.houseNumber + data << obj.id + data << obj.type// 物业类型 + data << ""//主朝向 + data << obj.floors//地上层数 + data << ""//地下层数 + data << obj.buildYear;//竣工日期 + data << ""//梯 + data << ""//户 + data << obj.households//总户数 + data << ""//景观 + }else{ + data << "" + data << "" + data << ""// 物业类型 + data << ""//主朝向 + data << ""//地上层数 + data << ""//地下层数 + data << ""//竣工日期 + data << ""//梯 + data << ""//户 + data << ""//总户数 + data << ""//景观 + } + + if(row['O']){ + def jsonObject1 = OpenApiClient.searchHouseLoudong(row['Q'], row['L'], row['M'], row['O']) + if (jsonObject1 && jsonObject1.result) { + if(jsonObject1.result.size()>=1){ + def obj = jsonObject1.result[0] + + data << obj.roomNum//户号 + data << obj.id//户ID + data << obj.direction//主 + data << obj.sight//景观 + if(obj.roomNum && obj.roomNum.contains("跃")){ + data << "跃式"//户型结构 + }else{ + data << ""//户型结构 + } + + } + } + } + } + rets << data + } + ExcelUtils.write(output, rets, "底层数据-楼栋-户") + } + + def addDetail(path, output){ + def datas = ExcelUtils.read(path, 0, 1) + def rows = [] + rows << ["输入城市","输入区域","输入地址","小区名称","小区ID","小区别名","小区地址","板块","街道","基准价","经纬度","竣工时间","开发商名称","绿化率","容积率","车位个数","物业公司","物业费","住宅幢数","总户数","车位配比","占地面积","是否学区","周边教育","周边交通","周边商业","周边医院","周边金融机构"] + for(def d : datas){ + if(d['Q']){ + def dist = CommonDatautils.findDistrictById(d['Q']) + if(dist){ + def row = [] + def jsonObject = OpenApiClient.districtDetail(dist.city, dist.region, dist.name) + if(jsonObject) { + + row << d['A'] + row << d['B'] + row << d['C'] + row << jsonObject.get("name")//小区名称 + row << d['Q']//小区ID + if(jsonObject.get("alias")){ + row << jsonObject.get("alias").join(",").replace('"',"")//小区别名 + }else{ + row << "" + } + + row << jsonObject.get("address")//小区地址 + row << jsonObject.get("commercialArea")//板块 + row << jsonObject.get("township")//街道*********** + row << jsonObject.get("avgPrice")//基准价 + row << jsonObject.get("lng").toString()+","+jsonObject.get("lat").toString()//经纬度 + row << jsonObject.get("finishBuildTime")//竣工时间 + row << jsonObject.get("developer")//开发商名称 + row << SeaweedUtils.formatVirescencePercent(jsonObject.get("greenPercentage"))//绿化率 + row << SeaweedUtils.formatCubagePercent(jsonObject.get("plotRatio"))//容积率 + row << SeaweedUtils.formatCarSiteCount(jsonObject.get("parking"))//车位个数 + row << jsonObject.get("propertyManagement")//物业公司 + row << jsonObject.get("managementFee")//物业费 + row << jsonObject.get("buildingCount")//住宅幢数 + row << SeaweedUtils.formatHouseholdCount(jsonObject.get("totalUnitCount"))//总户数 + row << SeaweedUtils.calcCarAndHouseholdRadio(jsonObject.get("parking"), jsonObject.get("totalUnitCount"))//车位配比********* + row << SeaweedUtils.formatAllArea(jsonObject.get("totalUnitCount"))//占地面积 + + } + + def facilityObj = OpenApiClient.facilitys(dist.city, dist.region, dist.name) + if(facilityObj && facilityObj.result){ + def result = facilityObj.result + if(result.keySchool){ + row << '是' + }else { + row << "" + } + + if(result.school){ + row << result.school.join(",").replace('"',"") + }else { + row << "" + } + + def bus = [] + if(result.bus){ + bus.addAll(result.bus) + } + if(result.metro){ + bus.addAll(result.metro) + } + + if(bus){ + row << bus.join(",").replace('"',"") + }else{ + row << "" + } + + if(result.supermarket){ + row << result.supermarket.join(",").replace('"',"") + }else{ + row << "" + } + + if(result.hospital){ + row << result.hospital.join(",").replace('"',"") + }else{ + row << "" + } + + if(result.bank){ + row << result.bank.join(",").replace('"',"") + }else{ + row << "" + } + } + rows << row + } + } + } + ExcelUtils.write(output, rows, "底层数据-小区") + } + + def createPaimai(path, output){ + def datas = ExcelUtils.read(path, 0, 1) + def rows = [] + rows << ["输入城市","输入区域","输入地址","小区名称","标的名称","楼栋","户号","面积","案例时间","类型","状态","成交单价","评估单价"] + def keys = [] + for(def d : datas){ + if(d['Q']) { + def dist = CommonDatautils.findDistrictById(d['Q']) + if (dist) { + def key = dist.city+"#"+dist.region+"#"+dist.name + if(keys.contains(key)){ + continue + } + keys << key + def jsonObject = OpenApiClient.searchPaimais(dist.city, dist.region, dist.name,10) + if(jsonObject && jsonObject.result){ + def results = jsonObject.result + for(def r: results){ + def row = [] + row << d['A'] + row << d['B'] + row << d['C'] + row << r.matchName + row << r.address + row << r.houseNum + row << r.roomNum + row << r.area + row << r.startDate + row << r.count + row << r.status + row << r.tradeAvgPrice + row << r.tradePrice + row << r.id + rows << row + } + println dist.name +" "+results.size() + } + } + } + } + ExcelUtils.write(output, rows, "拍卖案例") + } + + def createHouse(path, output){ + def datas = ExcelUtils.read(path, 0, 1) + def rows = [] + rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"] + def keys = [] + for(def d : datas) { + if (d['Q']) { + def key = d['Q'] + if (keys.contains(key)) { + continue + } + keys << key + def jsonObject = OpenApiClient.houseList(d['Q'],"20210501",20) + if(jsonObject && jsonObject.result) { + def results = jsonObject.result + for (def r : results) { + def row = [] + row << "挂牌" + row << r.districtName + if(r.estateType){ + row << r.estateType + }else{ + if(r.purpose){ + row << r.purpose + }else{ + row << "" + } + } + + if(r.room){ + def roomStr = r.room+"室" + if(r.hall){ + roomStr = roomStr +r.hall+"厅" + } + row << roomStr + }else{ + row << "" + } + row << r.area + row << r.fitment + row << r.pDate + row << "" + row << r.avgPrice + row << r.totalPrice + if(r.floor){ + row << r.floor + }else{ + row << "" + } + if(r.height){ + row << r.height + }else{ + row << "" + } + + row << r.direction + row << SourceNameUtil.getResourceName(r.root) + rows << row + } + + println d['Q'] +" "+results.size() + } + } + } + ExcelUtils.write(output, rows, "挂牌案例") + } + + def createEvaluation(path, output){ + def datas = ExcelUtils.read(path, 0, 1) + def rows = [] + rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"] + def keys = [] + for(def d : datas) { + if (d['Q']) { + def key = d['Q'] + if (keys.contains(key)) { + continue + } + keys << key + def jsonObject = OpenApiClient.evaluaionList(d['Q'],"20210501",20) + if(jsonObject && jsonObject.result) { + def results = jsonObject.result + for (def r : results) { + def row = [] + row << "评估" + row << r.commName + row << "" + + if(r.roomcnt){ + def roomStr = r.roomcnt+"室" + if(r.livingroomcnt){ + roomStr = roomStr +r.livingroomcnt+"厅" + } + row << roomStr + }else{ + row << "" + } + row << r.buildArea + row << r.fitment + row << r.cDate.substring(0, r.cDate.indexOf("T")) + row << "" + row << r.unitPrice + row << r.totalPrice + if(r.floor){ + row << r.floor + }else{ + row << "" + } + if(r.floorcnt){ + row << r.floorcnt + }else{ + row << "" + } + + if(r.houseaspect && !r.houseaspect.contains("暂")){ + row << r.houseaspect + }else{ + row << "" + } + + row << "房价网" + rows << row + } + + println d['Q'] +" "+results.size() + } + } + } + ExcelUtils.write(output, rows, "评估案例") + } + + + def createDealPrice(path, output){ + def datas = ExcelUtils.read(path, 0, 1) + def rows = [] + rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"] + def keys = [] + for(def d : datas) { + if (d['Q']) { + def key = d['Q'] + if (keys.contains(key)) { + continue + } + keys << key + def jsonObject = OpenApiClient.searchDealPrice(d['Q'],"20210501",20) + if(jsonObject && jsonObject.result) { + def results = jsonObject.result + for (def r : results) { + def row = [] + row << "成交" + row << r.districtName + if(r.type){ + row << r.type + }else{ + row << "" + } + + if(r.room){ + def roomStr = r.room+"室" + if(r.hall){ + roomStr = roomStr +r.hall+"厅" + } + row << roomStr + }else{ + row << "" + } + row << r.area + row << r.fitment + row << r.tradeDate.substring(0, r.tradeDate.indexOf("T")) + row << "" + row << r.avgPrice + row << r.totalPrice + if(r.floor){ + row << r.floor + }else{ + row << "" + } + if(r.height){ + row << r.height + }else{ + row << "" + } + + if(r.direction && !r.direction.contains("暂")){ + row << r.direction + }else{ + row << "" + } + + row << r.source + rows << row + } + + println d['Q'] +" "+results.size() + } + } + } + ExcelUtils.write(output, rows, "成交案例") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 239c9f83c..d6ab63366 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -830,6 +830,7 @@ class CommonDatautils { keys.put("name_cat", 1); keys.put("finish_building_date", 1) keys.put("mapping_alias", 1) + keys.put("sid", 1) DSCursor cursor = getDataStoreConnection().find(q, keys); if(cursor){ for(def obj : cursor){ @@ -3166,5 +3167,22 @@ class CommonDatautils { } return avg_price; } + + static def findCityAvgRentRadio(def city, def weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("weekend", weekend) + + def dbcursor = SQMETRE_RENT_COLL.find(q) + def avg_price = null + def radios = [] + for(def d : dbcursor){ + if(d.price_to_rent){ + radios << d.price_to_rent.toInteger() + } + } + def avg_radios = (int)(radios.sum()/radios.size()) + return avg_radios; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/common/DataHandleUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/common/DataHandleUtils.groovy index 7d1ef8018..7fe3204dd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/common/DataHandleUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/common/DataHandleUtils.groovy @@ -16,9 +16,7 @@ import org.apache.commons.lang.math.RandomUtils; class DataHandleUtils { - static main(args){ - - } + static def getStandardRegion(def regionMap, def region){ def newRegion = null; @@ -94,6 +92,8 @@ class DataHandleUtils { BigDecimal deSource = new BigDecimal(dSource); return deSource.setScale(scale, BigDecimal.ROUND_HALF_UP).doubleValue(); } + + static def trimChar(String c) { try { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictAroundFacilitiesBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictAroundFacilitiesBatch.groovy index 2a2f5890f..63d7f841f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictAroundFacilitiesBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictAroundFacilitiesBatch.groovy @@ -35,7 +35,7 @@ class DistrictAroundFacilitiesBatch extends BaseBatch { @Override public void process(Object args) { - def cities = ["北京", "上海"] + def cities = ["北京", "上海"]// for (def city : cities) { def ret = CommonDatautils.getAllDistricts("district", city) // def ret = getAllDistrictBaseInfo("上海", '闵行', '万源城', "district"); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index b49996401..68494ea8b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -42,17 +42,17 @@ class YouDaDataParser { static def new_queryUrl = "http://192.168.0.228:9200/new_deal_price_lansi/doc/_search" - static def esf_targetUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/doc" + static def esf_targetUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/_doc" static def esf_queryUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/doc/_search" public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2022-02-01" + static def statDate = "2022-03-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo -// task.esfParser();//友达二手房存入mongo + task.esfParser();//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es task.esfhouseToSaveEs();//二手房保存到es // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 @@ -285,7 +285,7 @@ class YouDaDataParser { def esfParser(){ - def files = ["D:\\数据\\友达\\2022年3月\\查询1.DBF"] + def files = ["D:\\数据\\友达\\2022年4月\\查询1.DBF"] def datas = [] for(def fileName : files){ def rows = readDBF(fileName); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/apiLog/ApiTimeOutAnalyBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/apiLog/ApiTimeOutAnalyBatch.groovy new file mode 100644 index 000000000..21dbfb7c8 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/apiLog/ApiTimeOutAnalyBatch.groovy @@ -0,0 +1,61 @@ +package com.fangjia.script.gj.apiLog + +import com.alibaba.excel.util.DateUtils +import com.fangjia.core.util.DateUtil +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.util.DingDingMessageUtil +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.Calendar; + +@BatchOwner(Watcher.GaoJuan) +class ApiTimeOutAnalyBatch extends BaseBatch{ + + DBCollection LOG = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.42", 52002) + + static main(args) { + runBatch(new ApiTimeOutAnalyBatch(), args); + } + + @Override + public void process(Object args) { + def uriCountMap = findTimeOutUri() + sendMessage(uriCountMap) + } + + def sendMessage(uriCountMap){ + def message = "【Api接口近5分钟超过5条请求时间超3秒提醒】\n超时接口:" + def total = message.length() + uriCountMap.each{uri,count-> + if(count>=5){ + message = message + uri +"\n" + } + } + if(message.length()>total){ + println message + def access_token = "1e25f620ea2be926b2441e4f30d67f37abdff2e14088b03b41b4082b11fc34ea" + DingDingMessageUtil.sendTextMessage(access_token, message) + } + } + + def findTimeOutUri(){ + def q = new BasicDBObject() + q.put("time", new BasicDBObject('$gte', 3000)) + def startTime = DateUtil.add(new Date(), Calendar.MINUTE, -5) + q.put("c_date", new BasicDBObject('$gte', startTime)) + def dbcursor = LOG.find(q) + def uriCountMap = [:] + for(def d : dbcursor){ + def count = uriCountMap.get(d.request_uri) + if(!count){ + count = 0 + } + uriCountMap.put(d.request_uri, count + 1) + } + return uriCountMap + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/apiLog/ApiUserCountAnalyBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/apiLog/ApiUserCountAnalyBatch.groovy new file mode 100644 index 000000000..16063d88b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/apiLog/ApiUserCountAnalyBatch.groovy @@ -0,0 +1,111 @@ +package com.fangjia.script.gj.apiLog + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.util.DingDingMessageUtil +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import groovy.sql.Sql + +@BatchOwner(Watcher.GaoJuan) +class ApiUserCountAnalyBatch extends BaseBatch{ + + static final dbFangjia = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia?autoReconnect=true&characterEncoding=utf-8&autoReconnect=true", "sa", "G00jia", "com.mysql.jdbc.Driver"); + def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + DBCollection AUTHORITY_COLL = MongoUtils.getCollectionByName2("fangjia", "authority.api_user_authority", lineUrl) + DBCollection VISIT_COLL = MongoUtils.getCollectionByName2("business", "api_visit_count", lineUrl) + + + static main(args) { + runBatch(new ApiUserCountAnalyBatch(), args); + } + + @Override + public void process(Object args) { + def rets = findAllUsers() + sendMessage(rets) + } + + + def sendMessage(def rets){ + def message = "【Api接口使用提醒】\n" + def total = message.length() + for(def r : rets){ + def type = "正式帐户" + if(r.type!=0){ + type = "测试帐户" + } + message = message + r.company+ " 用户ID:"+r.user_id+"(${type}),有 "+r.uris.size()+" 个接口使用率已超过80%,接口有:"+r.uris.join(",")+"\n" + } + if(message.length()>total){ + println message + def access_token = "1e25f620ea2be926b2441e4f30d67f37abdff2e14088b03b41b4082b11fc34ea" + DingDingMessageUtil.sendTextMessage(access_token, message) + } + + } + + def findAllUsers(){ + def sql = "select id,COMPANY,is_test from api_user where dead_time>='"+DateUtils.formatDateByPattern(new Date(),'yyyy-MM-dd')+"'" + def rows = dbFangjia.rows(sql) + def rets = [] + for(def row : rows){ + def obj = findResources(row.id) + if(obj){ + def map = obj.total_count_map + def uris = [] + map.each{k,v-> + //def visit_count = findVisitCount(row.id, k) + def total_visit_count = findAllVisitCount(row.id, k) + println row.id +" "+k+" "+v+" "+total_visit_count + if(v && total_visit_count){ + def precent = v*0.8; + if(total_visit_count>=precent){ + uris << k + } + } + } + if(uris){ + rets << ["user_id":row.id,"company":row.company,"uris":uris,"type":row.is_test] + } + } + } + + return rets; + } + + def findAllVisitCount(userId, uri){ + def q = new BasicDBObject() + q.put("user_id", userId.toString()) + q.put("uri", uri) + q.put("visit_date", new BasicDBObject('$lte',DateUtils.truncateDay(new Date()))) + def dbcursor = VISIT_COLL.find(q) + def total = 0 + for(def obj : dbcursor){ + total = total +obj.visit_count + } + return total + } + + + def findVisitCount(userId, uri){ + def q = new BasicDBObject() + q.put("user_id", userId.toString()) + q.put("uri", uri) + q.put("visit_date", DateUtils.truncateDay(new Date())) + def obj = VISIT_COLL.findOne(q) + if(obj){ + return obj.visit_count + } + return null + } + + def findResources(def userId){ + def q = new BasicDBObject() + q.put("user_id", userId) + return AUTHORITY_COLL.findOne(q) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy index 3e4981856..23513b80b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy @@ -50,4 +50,32 @@ class FileUtils { } out.close(); } + + static def createTxtFile(ret, fileName, charset, isAppend){ + File file = new File(fileName); + if (!file.getParentFile().exists()) { + boolean result = file.getParentFile().mkdirs(); + if (!result) { + System.out.println("创建目录失败"); + } + } + + OutputStream out = new FileOutputStream(fileName, isAppend); + def sb = new StringBuffer(); + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + def index = 0 + for(def row : ret){ + if(row){ + index++ + sb.append(row) + sb.append("\n"); + } + } + if(sb.length()>0){ + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + } + out.close(); + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy index 9f84f6479..418b6c102 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy @@ -25,11 +25,11 @@ class DataMatch { static main(args) { def task = new DataMatch() // task.run1() - task.run2() +// task.run2() // task.run3() // println task.findAddress("昌州街道棠香路") -// task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx") + task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx") } def run1(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDetailCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDetailCrawl.groovy new file mode 100644 index 000000000..d4f3001b3 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDetailCrawl.groovy @@ -0,0 +1,246 @@ +package com.fangjia.script.gj.crawl + +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.jsoup.Jsoup +import org.openqa.selenium.By +import org.openqa.selenium.WebDriver +import org.openqa.selenium.WebElement +import org.openqa.selenium.chrome.ChromeDriver + +class AnjukeDetailCrawl { + DBCollection detailColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_detail") + static main(args) { + def task = new AnjukeDetailCrawl(); + task.run(); + } + def run(){ + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + def url = "https://chongqing.anjuke.com/community/?from=navigation" + WebDriver driver = new ChromeDriver() + def regions = getRegions(driver, url) + getAllRegions(driver, regions) + + } + + + def getAllRegions(driver, regions){ + for(def url : regions){ + driver.get(url) + def doc = Jsoup.parse(driver.getPageSource()); + def total = getTotal(doc) + if(total) { + if (total > 1250) { + + } else { + def links = getAllLinks(url, driver) + parse(links,driver) + } + } + } + } + + def getAllLinks(url, driver){ + def stop = false + def urls = [] + while(!stop) { + def doc = Jsoup.parse(driver.getPageSource()); + def details = JQueryUtils.select(".list-cell a.li-row", doc) + + for (def d : details) { + urls << d.attr("href") + } + def nextEle = JQueryUtils.select(".next", doc) + if (nextEle) { + driver.findElement(By.cssSelector(".next")).click() + }else{ + stop = true + } + if(urls.size()>20){ + stop = true + } + } + + return urls + } + + def getTotal(doc){ + + def totalText = JQueryUtils.selectText(".total-info", doc) + if(totalText){ + def total = totalText.substring(totalText.indexOf("到")+1, totalText.indexOf("个")).trim() + if(total && total.isNumber()){ + return total.toInteger() + } + } + return null; + } + + def getRegions(driver, url){ + driver.get(url) + Thread.sleep(3000) + def doc = Jsoup.parse(driver.getPageSource()); +// def loginIfm = JQueryUtils.select("#iframeLoginIfm", doc) +// if(loginIfm){ +// def href = loginIfm[0].attr("src") +// driver.get(href) +// driver.findElementById("pwdTab").click() +// WebElement elemUsername = driver.findElement(By.id("pwdUserNameIpt")); //获取用户名框 +// WebElement elemPassword = driver.findElement(By.id("pwdIpt")); //获取密码框 +// WebElement btn = driver.findElement(By.cssSelector("input[class='submit-btn']")); //获取点击登陆按钮。 +// elemUsername.sendKeys("17740818975"); +// elemPassword.sendKeys("12qwaszx"); +// btn.submit(); // 点击行为; +// +// def regionEles = JQueryUtils.select(".region-parents li a", doc) +// def urls = [] +// for(def e : regionEles){ +// urls << e.attr("href") +// } +// return urls; +// } + def regionEles = JQueryUtils.select(".region-parents li a", doc) + def urls = [] + for(def e : regionEles){ + urls << e.attr("href") + } + return urls; + } + + def parse(def links, driver){ + def rets = [] + for(def link : links){ + def district = parsing(link, driver) + if(district){ + rets << district + + if (rets.size() % 100 == 0) { + def add= saveData(rets) + println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + rets = [] + } + } + } + if (rets.size()> 0) { + def add= saveData(rets) + println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + } + } + + + def saveData(def results){ + def add = 0 + for(def ret : results){ + def q = new BasicDBObject() + q.put("url", ret.url) + def dbo = detailColl.findOne(q); + + DBObject obj = new BasicDBObject(); + ret.each{k,v-> + obj.put(k,v) + } + + if(dbo){ + obj.m_date = new Date() + detailColl.update(q, new BasicDBObject('$set', obj)) + }else{ + add++ + obj.c_date = new Date() + obj.m_date = new Date() + try{ + detailColl.save(obj) + }catch(Exception e){ + println obj.url + } + } + } + return add + } + + + + def parsing(url, driver){ + + def district = [:] + def stop = false + def count = 0 + while(!stop){ + driver.get(url) + def doc = Jsoup.parse(driver.getPageSource()); + def headers = JQueryUtils.select(".crumbs a", doc) + if(headers){ + if (headers && headers.size() >= 3) { + district.city = headers[1].text().replace("小区", "") + district.region = headers[2].text().replace("小区", "") + district.name = headers[headers.size() - 1].text().replace("二手房", "") + } + + + district.address = JQueryUtils.selectText("div.column-1:contains(小区地址) .value", doc) + + def priceStr = JQueryUtils.selectText(".average", doc) + + if (priceStr && priceStr.isNumber()) { + district.avg_price = priceStr.toInteger() + } + + def infoDiv = JQueryUtils.select(".info-list", doc) + district.estate_type = getValue(infoDiv.select(".column-2:contains(物业类型) .value").text()) + district.estate_charge = getValue(infoDiv.select(".column-2:contains(物业费) .value").text()) + district.estate_company = getValue(infoDiv.select(".column-1:contains(物业公司) .value").text()) + district.developer = getValue(infoDiv.select(".column-1:contains(开发商) .value").text()) + + def houseCountStr = infoDiv.select(".column-2:contains(总户数) .value").text() + if (houseCountStr) { + houseCountStr = houseCountStr.replace("户", "") + if (houseCountStr) { + district.house_count = houseCountStr.toInteger() + } + } + + district.finish_date = getValue(infoDiv.select(".column-2:contains(竣工时间) .value").text()) + district.build_area = getValue(infoDiv.select(".column-2:contains(总建面积) .value").text()) + def carSiteStr = getValue(infoDiv.select(".column-2:contains(停车位) .value").text()) + if(carSiteStr){ + if(carSiteStr.contains("(")){ + carSiteStr = carSiteStr.substring(0, carSiteStr.indexOf("(")) + } + if(carSiteStr && carSiteStr.isNumber()){ + district.car_site = carSiteStr + } + } + district.volumetric_rate = getValue(infoDiv.select(".column-2:contains(容积率) .value").text()) + district.greening_rate = getValue(infoDiv.select(".column-2:contains(绿化率:) .value").text()) + district.build_type = getValue(infoDiv.select(".column-2:contains(建筑类型) .value").text()) + district.property_life = getValue(infoDiv.select(".column-2:contains(产权年限) .value").text()) + district.car_site_charge = getValue(infoDiv.select(".column-1:contains(停车费) .value").text()) + district.url = url + district.c_date = new Date() + district.m_date = new Date() + district.root = "anjuke.com" + + stop = true; + }else{ + Thread.sleep(3000) + count++ + } + if(count>10){ + stop = true; + } + } + + return district + } + + String getValue(String s){ + if(s && s.contains("暂无")){ + return null + } + return s + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/CommonCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/CommonCrawl.groovy new file mode 100644 index 000000000..f19c4cd3c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/CommonCrawl.groovy @@ -0,0 +1,29 @@ +package com.fangjia.script.gj.crawl + +import org.jsoup.Jsoup + +class CommonCrawl { + static def getDocument(url, driver){ + def stop = false + def error = 0 + def doc = null + while(!stop){ + try{ + driver.get(url) + doc = Jsoup.parse(driver.getPageSource()); + if(doc){ + stop = true; + }else{ + error++ + Thread.sleep(5000) + } + }catch(Exception e){ + error++ + } + if(error>=10){ + stop = true; + } + } + return doc + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/TempMatchDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/TempMatchDistrict.groovy index a36cf12a7..13b8fdde9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/TempMatchDistrict.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/TempMatchDistrict.groovy @@ -17,7 +17,7 @@ class TempMatchDistrict { } def stat(){ - def citys = ["哈尔滨","齐齐哈尔","大庆","牡丹江","佳木斯","绥化","鸡西","七台河","双鸭山","鹤岗","黑河","伊春","大兴安岭"] + def citys = ["广州","深圳","佛山","东莞","中山","湛江","惠州","江门","珠海","汕头","揭阳","茂名","清远","肇庆","潮州","河源","韶关","阳江","梅州","云浮","汕尾"] def total = 0 for(def city : citys){ def count = CommonDatautils.findAllDistrictCount("district", city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 44f6d4a9b..12d2f77b0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -83,6 +83,23 @@ class FangjiaClientUtils { return jsonObj; } + public static def arroundPrice(city,address){ + def jsonObj = null + def content = null + try { + address = address.replace("{","") + // def url = "http://192.168.0.19:39080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8") + def url = "http://192.168.0.100:38080/FangJiaDataService/evaluate?address="+URLEncoder.encode(address,"UTF-8")+"&city="+URLEncoder.encode(city,"UTF-8")+"&maxNum=10&radius=1000&size=100" +// def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8") + content = new HttpRequest().newGet(url, null, "utf-8") + jsonObj = JSONObject.fromObject(content) + } catch (Exception e) { + println address +" "+content + e.printStackTrace(); + } + return jsonObj; + } + public static Map JsonToMap(JSONObject json) { Map columnValMap = new HashMap(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy index e6046e703..bd387ed77 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -58,6 +58,8 @@ class DistrictIndexBatch extends BaseBatch{ def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + static def month = DateUtils.formatDateByPattern(new Date(),"yyyy-MM") + static main(args) { runBatch(new DistrictIndexBatch(), args); } @@ -71,7 +73,7 @@ class DistrictIndexBatch extends BaseBatch{ def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def citys = CommonDatautils.findAllCity(); for(def city : citys){ - def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2"]) + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2","household_count"]) def code = cityCodeDatas.get(city) def city_avg_total_price = null if(city!="上海"){ @@ -84,7 +86,7 @@ class DistrictIndexBatch extends BaseBatch{ city_avg_total_price = 531.2615913 } if(city_avg_total_price){ - calcute(city, dists, code, city_avg_total_price) + // calcute(city, dists, code, city_avg_total_price) calcuteIndex(city, dists, code, city_avg_total_price) } @@ -131,7 +133,7 @@ class DistrictIndexBatch extends BaseBatch{ map.total_price = total_price //map.comp_value = potentiality //map.avg_total_price = total_price_index - map.month = "2022-03" + map.month = month map.c_date = DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T") //datas << [d.city, d.region, d.name, total_price_index, avg_charge_index, rank_index, avg_cost_index,potentiality] @@ -229,7 +231,7 @@ class DistrictIndexBatch extends BaseBatch{ map.comp_value = potentiality map.avg_total_price = total_price_index - map.month = "2022-03" + map.month = month map.m_date = DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T") map.finish_date = year @@ -240,6 +242,15 @@ class DistrictIndexBatch extends BaseBatch{ if(rMap.sell_avg_price && sqmetre_rent){ map.price_to_rent = (rMap.sell_avg_price/sqmetre_rent).toInteger() } + if(d.household_count){ + map.household_count = d.household_count + } + if(d.lat){ + def lMap = [:] + lMap.lon = d.lng/10000000000.0 + lMap.lat = d.lat/10000000000.0 + map.location = lMap + } def newUrl = targetUrl +"_doc/"+map.id httpSaveData(newUrl, map) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meituan/MatchTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meituan/MatchTest.groovy new file mode 100644 index 000000000..6b4872d19 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meituan/MatchTest.groovy @@ -0,0 +1,354 @@ +package com.fangjia.script.gj.meituan + +import com.fangjia.core.RegionData +import com.fangjia.core.util.DateUtil +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import com.fangjia.script.gj.paimai.DataUtil +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.GeoUtils +import com.fangjia.script.util.OpenApiClient +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import org.bson.types.ObjectId + +class MatchTest { + static main(args) { + def task = new MatchTest() +// task.run() +// task.addNum() + task.dongguang() +// task.meituanDel() + } + + def meituanDel(){ + def path = "D:\\数据\\美团\\美团需入库小区合并后.xlsx" + def index = 0 + ExcelUtils.modify(path, { row -> + def flag = row['O'] + if(flag){ + def city = row['F'] + def region = row['G'] + def name = nameFileter(row['H']) + def obj = null + if(city == "东莞"){ + obj = CommonDatautils.findDistrict(city, null, name, "district") + }else{ + obj = CommonDatautils.findDistrict(city, region, name, "district") + } + if(obj){ + index++ + println index +" "+ city +' '+ region+" "+name + def udata = new BasicDBObject() + udata.put("name_cat", 11) + udata.put("status", 1) + udata.put("m_date",new Date()) + + def q = new BasicDBObject() + q.put("_id", new ObjectId(obj._id.toString())) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + } + } + },0); + } + + def dongguang(){ + def path = "D:\\数据\\美团\\美团东莞.xlsx"//美团金融小区测试list-匹配.xlsx + ExcelUtils.modify(path, { row -> + def region = null + if(row['B'] && row['B']=="NULL" ){ + region = null + } + def name = nameFileter(row['H']) + def address = row['E'] + if (name && name.contains("湖畔新家")) { + + def obj = CommonDatautils.findDistrict("东莞", "东莞", name, "district") + if(obj){ + def json = GaoDeApiUtils.geo(row['F'], address) + if(json && json.count>=1){ + def m = json.geocodes[0] + if(m){ + def twinshop = getStreetInfo(m.location) + if(twinshop){ + def newRegion = RegionData.formatRegion("东莞",twinshop) + if(newRegion){ + def dbObj = CommonDatautils.findDistrict("东莞", newRegion, name, "district") + if(dbObj){ + println "delete" + DbUtils.seaweed.remove(new BasicDBObject("_id",obj._id)) + }else{ + println "update" + def udata = new BasicDBObject() + udata.put("region", newRegion) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata)) + } + } + } + // println address +" "+twinshop + } + } + } + } + },0) + + } + + def getStreetInfo(def glocation){ + def township = null + if(glocation){ + def jsonObject = GaoDeApiUtils.geocode(glocation) + if(jsonObject && jsonObject.regeocode) { + def addressComponent = jsonObject.regeocode.addressComponent + if(addressComponent){ + township = addressComponent.township + } + } + } + return township; + } + + def addNum(){ + def path = "D:\\数据\\美团\\美团金融小区测试list-所有数据.xlsx"//美团金融小区测试list-匹配.xlsx + def index = 0 + ExcelUtils.modify(path, { row -> + index++ + row['A'] = index + },0) + } + + def run() { + def citys = ["潮州","金华","惠州","东莞","苏州","桂林","廊坊","临沧","通辽","雅安"]// + def weekend = DateUtils.getFirstDayOfWeek(new Date()); + def cityLevelMap = [:] + def radioMap = [:] + for(def city : citys){ + +// def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1)); +// def avg_radio = CommonDatautils.findCityAvgRentRadio(city, lastMonthDay) +// radioMap.put(city, avg_radio) + + + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region","name"]) + def priceMap = CommonDatautils.queryDistrictPrice(city, weekend) + def datas = addPriceData(dists, priceMap); + datas = calcuteLevel(datas); + cityLevelMap.put(city, ["price":priceMap, "level":datas]) + println city +" 楼盘评级同步完成" + + } + + + def path = "D:\\数据\\美团\\美团金融小区测试list-所有数据-评级.xlsx"//美团金融小区测试list-所有数据-评级.xlsx + ExcelUtils.multiThreadingModify(path, { row -> + def region = null + if(row['C'] && row['C']=="NULL" ){ + region = null + } + def name = row['D'] + def address = row['F'] + def city = RegionData.formatCity(row['B']) + + if(name && !row['N']){ + if(city){ + def map = cityLevelMap.get(city) + def rMap = null + if(map){ + rMap = getArroundDistrictPriceByAddress(city, address, map.level) + } + + if(rMap){ + if(rMap.rank){ + row['Q'] = rMap.rank + } + if(rMap.level){ + row['P'] = rMap.level + } + } + } + } + + if (name && row['N']) { +//// def jsonObject = OpenApiClient.addressMatch(row['A'], address, null, true, region, name) +// def jsonObject = OpenApiClient.addressMatch(row['A'], name, null, true, region, null) +// if (jsonObject != null && jsonObject.precise != -1) { +// row['F'] = jsonObject.get("city") +// row['G'] = jsonObject.get("district") +// row['H'] = jsonObject.get("name") +// row['I'] = jsonObject.get("propertyType") +// row['J'] = jsonObject.get("map") +// row['K'] = jsonObject.get("lng") +// row['L'] = jsonObject.get("lat") +// } + + if(row['G'] && row['H'] && row['I']){ +// def priceObj = CommonDatautils.findDistrictPrice(row['F'], row['G'], row['H']) +// if(priceObj){ +// row['M'] = priceObj.avg_price +// } +// def sqmetreRentPrice = CommonDatautils.findNewSqmetreRent(row['F'], row['G'], row['H']) +// if(sqmetreRentPrice){ +// row['N'] = sqmetreRentPrice +// } + + if(row['N'] && !row['O']){ + def avg_rent_radio = radioMap.get(city) + row['O'] = (int)(row['N']/avg_rent_radio) + } + } + + } + }) + } + + def getArroundDistrictPriceByAddress(def city, def address, def levels){ + def json = GaoDeApiUtils.geo(city, address) + def location = null + def detail = new BasicDBObject() + def level = null + def rank = null + if(json && json.count>=1){ + def m = json.geocodes[0] + if(m){ + location = m.location + level = m.level + } + } + if(location){ + def arrs = location.split(",") + detail.lng = arrs[0].toString().toDouble()*10000000000L + detail.lat = arrs[1].toString().toDouble()*10000000000L + } + if(detail.lng){ + detail.city = city + + List list = DbUtils.findAroundDistrictByRadius(detail, 3000) + if(list && list.size()>10){ + list = sortByDistance(detail, list); + list = list[0..9] + } + def arroundAvgPrice = getArroundAvgPrice(list) + if(arroundAvgPrice){ + + def mod = Integer.MAX_VALUE; + for(def d : levels){ + def newMod = java.lang.Math.abs( d.avg_price - arroundAvgPrice) + if(newMod o2.distance) + return 1;//第二个元素(o1)比第一个元素(o2)大,返回-1 + if(o1.distance == o2.distance) + return 0; + return -1; + } + }) + return list; + } + + def addPriceData(dists, priceMap){ + def newLists = [] + for(def d : dists){ + def price = priceMap.get(d.city+"##"+d.region+"##"+d.name); + if(price){ + d.avg_price = price; + newLists << d; + } + } + if(newLists){ + newLists.sort {a,b-> + def v1 = a.avg_price + def v2 = b.avg_price + return v1.compareTo(v2) + } + } + return newLists; + } + + def calcuteLevel(def datas){ + def total = datas.size() + def v = (int)(total/10) + def start = 0; + def end = 0; + def map = [:] + for(int i=1;i<=10;i++){ + end = end + v; + if(i == 10){ + map.put(i, [start, total]) + }else{ + map.put(i, [start, end]) + } + start = end; + } + for(def d=0;dm && d<=n){ + dist.level = 10 - o.key + 1 + break + } + } + } + } + return datas; + } + + def getArroundAvgPrice(def list){ + def prices = [] + for(def l : list){ + def priceObj = CommonDatautils.findDistrictPrice(l.city, l.region, l.name) + if(priceObj){ + prices << priceObj.avg_price + } + } + if(prices){ + def avg_price = (prices.sum()/prices.size()).toInteger() + return avg_price; + } + return null + } + + def nameFileter(def name){ + name = name.replace("·","").replace(".","").replace("|","") + name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期") + name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区") + return name; + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index d54ad9fb3..a6b739898 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -45,9 +45,21 @@ public class BeiKeCrawlByChrome { System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); System.setProperty("http.nonProxyHosts", "localhost"); WebDriver driver = new ChromeDriver() - def cityMap = getAllCityUrls(driver) + def citys = getAllCityUrls(driver) + citys = citys.reverse() + def start = 0 +// for(def i=0;i + for(def m : citys){ //def detailUrls = getPageUrls(driver, cityUrl) - def detailUrls = getPageDeail(driver, city, cityUrl) - println cityUrl +" 详情链接:"+detailUrls.size() + def detailUrls = getPageDeail(driver, m.city, m.url) + println m.city +" 详情链接:"+detailUrls.size() // crawlDetail(driver, DEAL_PRICE_COLL, detailUrls) } } @@ -141,7 +153,7 @@ public class BeiKeCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd") if(dealDate && dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e) @@ -150,6 +162,10 @@ public class BeiKeCrawlByChrome { break; } } + if(page>=100){ + stop = true; + break; + } page++; }else{ stop = true; @@ -182,7 +198,7 @@ public class BeiKeCrawlByChrome { } def areaStr = arrs[2].replace("平米", "") - if (areaStr) { + if (areaStr && areaStr.isNumber()) { detail.area = areaStr.toDouble() } @@ -337,17 +353,21 @@ public class BeiKeCrawlByChrome { driver.get("https://www.ke.com/city/") Thread.sleep(8000) def cityDoc = Jsoup.parse(driver.getPageSource()); - def cityEles = JQueryUtils.select(".city_province a", cityDoc) - def cityMap = [:] + def cityEles = JQueryUtils.select(".city_list_ul .city_province a", cityDoc) + def list = [] if(cityEles){ for(def e : cityEles){ def href = e.attr("href") - if(e.text().contains("福州")){ - cityMap.put(e.text(), "https:"+href) + if(e.text().contains("重庆")){ + def cityMap = [:] + cityMap.city = e.text() + cityMap.url = "https:"+href + list << cityMap + //cityMap.put(e.text(), "https:"+href) } } } - return cityMap + return list } def getLogin(driver){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy index afaae338f..fb358e503 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy @@ -42,7 +42,6 @@ System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Loca System.setProperty("http.nonProxyHosts", "localhost"); - WebDriver driver = new ChromeDriver() def proviceMap = getDocument(driver); def citys = getCity(driver,proviceMap) @@ -59,9 +58,11 @@ def getDetailLinks(driver, cityList){ citys.each{city,id-> def urls = [] if(!zxcitys.contains(cityMap.provice)){ - driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&city=${id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport="); + def url = "https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&city=${id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport=" + getDocument(url, driver) }else{ - driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport="); + def url = "https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport=" + getDocument(url, driver) } def doc = Jsoup.parse(driver.getPageSource()); @@ -89,7 +90,7 @@ def getDetailLinks(driver, cityList){ } } - def rets = batchParse(urls); + def rets = batchParse(urls, driver); saveToDB(rets); println city +" "+urls.size() } @@ -124,7 +125,7 @@ def getCity(driver, proviceMap){ for(def c : cityEles){ def cityid = c.attr("data-cityid") def city = c.text() - if(!["石家庄市","太原市","长治市","朔州市","廊坊市","衡水市","唐山市","秦皇岛市","邯郸市","邢台市","保定市","沧州市","张家口市","承德市"].contains(city)){ + if(!["石家庄市","太原市","齐齐哈尔市","哈尔滨市","鸡西市","长治市","朔州市","廊坊市","衡水市","唐山市","秦皇岛市","邯郸市","邢台市","保定市","沧州市","张家口市","承德市"].contains(city)){ cityMap.put(city, cityid) } @@ -149,7 +150,7 @@ def getCity(driver, proviceMap){ def getDocument(driver){ - driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=9&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&canLoan=&isRestricted="); + driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=1&canLoan=&isRestricted=&insuranceSupport="); WebDriverWait wait = new WebDriverWait(driver,20) wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("lots-list")))); def doc = Jsoup.parse(driver.getPageSource()); @@ -159,7 +160,7 @@ def getDocument(driver){ for(def ele : proviceEles){ def provice = ele.text() def value = ele.attr("data-proviceid") - if(value && provice!="北京市" && provice!="天津市"){ + if(value && provice!="北京市" && provice!="天津市" && provice!="吉林省" && provice!="辽宁省" && provice!="内蒙古自治区" && provice!="山西省" && provice!="河北省" && provice!="江苏省" && provice!="黑龙江省"){ provinceMap.put(provice, value) } } @@ -167,8 +168,8 @@ def getDocument(driver){ return provinceMap; } -def batchParse(def urls){ - WebDriver driver = new ChromeDriver() +def batchParse(def urls, driver){ + def ret = [] for(def map :urls){ def doc = getDocument(map.url, driver) @@ -280,7 +281,7 @@ def parse(province, city, url,doc){ result.lookCount = JQueryUtils.selectText(".look_count span", doc) def startDateStr = JQueryUtils.selectText("span:contains(拍卖会定于)", doc) - if(startDateStr){ + if(startDateStr && startDateStr.endsWith("定于") && startDateStr.endsWith("日")){ startDateStr = startDateStr.toString().substring(startDateStr.indexOf("定于")+2, startDateStr.indexOf("日")+1).trim() } @@ -329,6 +330,13 @@ def parse(province, city, url,doc){ result.tradeAvgPrice = (int) (result.tradePrice / result.area) } + if(result.finishDate){ + result.finishDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.finishDate, "2000-01-01") + } + if(result.startDate){ + result.startDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.startDate, "2050-01-01") + } + result.content = DataUtil.zip(doc.toString()) result.introduce = JQueryUtils.select([".d-block table:eq(1)"], doc).toString() result.url = url diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataUtil.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataUtil.groovy index 8cce171c6..7e81c3afd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataUtil.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataUtil.groovy @@ -16,6 +16,27 @@ import java.util.regex.Matcher import java.util.regex.Pattern class DataUtil { + + + static def checkMinDate(curDate, checkDate){ + if(curDate){ + def minDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(checkDate,"yyy-MM-dd") + if(curDate.getTime()maxDate.getTime()){ + return null + } + } + return curDate + } static def getAreaNumber(def input, def maxArea) { Pattern pattern = Pattern.compile('\\d+(.\\d+)?(平方米|㎡|m2|方米|平)') Matcher matcher = pattern.matcher(input) @@ -112,10 +133,10 @@ class DataUtil { if (list.size() >= 1) { String s1 = list.get(0) if (s1.contains("万元")) { - s1 = s1.toString().replace("起拍价:", "").replace("万元", "").replace(",", "").replace(",", "") + s1 = s1.toString().replace("起拍价:", "").replace("万元", "").replace(",", "").replace(",", "").replace(" ","") return (int) (s1.toDouble() * 10000) } else { - s1 = s1.toString().replace("起拍价:", "").replace("元", "").replace(",", "").replace(",", "") + s1 = s1.toString().replace("起拍价:", "").replace("元", "").replace(",", "").replace(",", "").replace(" ","") if (s1.isInteger()) { return s1.toInteger() } else { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/IcbcCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/IcbcCrawl.groovy new file mode 100644 index 000000000..b432108a2 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/IcbcCrawl.groovy @@ -0,0 +1,348 @@ +package com.fangjia.script.gj.paimai + +import com.fangjia.core.RegionData +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import javafx.scene.input.DataFormat +import net.sf.json.JSONObject +import org.jsoup.Jsoup +import org.openqa.selenium.By +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.ChromeDriver + +class IcbcCrawl { + DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai") + + static main(args) { + def task = new IcbcCrawl(); + task.run(); + } + + def run(){ + def url = "https://gf.trade.icbc.com.cn/searchproducts/pv.jhtml?resultLevel=1&FilterObjectName=%E8%B5%84%E4%BA%A7%E5%A4%84%E7%BD%AE%E5%B9%B3%E5%8F%B0&filterObjectId=8&category_Type=3&categoryId_level=&subjectChildType=1#this" + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + WebDriver driver = new ChromeDriver() + driver.get(url) + driver.findElementById("category_SM310000000000000006").click() + Thread.sleep(3000) + + + def urls = getAllDetailLinks(driver) + println "共:"+urls.size() + + detailCrawling(urls, driver) + + } + + def getAllDetailLinks(driver){ + def stop = false + def urls = [] + while (!stop){ + def doc = Jsoup.parse(driver.getPageSource()); + def eles = JQueryUtils.select(".tabContent5 .ymzs_sp", doc) + if(eles){ + for(def e : eles){ + def href = e.select(".li_2_name a").attr("href") + urls << "https://gf.trade.icbc.com.cn"+href + } + def nextPage = JQueryUtils.select("a:contains(下一页)", doc) + if(nextPage){ + driver.findElement(By.linkText("下一页")).click(); + }else{ + stop = true + } + }else{ + stop = true + } +// if(urls.size()>=10){ +// stop = true +// } + } + return urls; + } + + def detailCrawling(def urls, driver){ + def rets = [] + for(def url : urls){ + driver.get(url) + Thread.sleep(5000) + def doc = Jsoup.parse(driver.getPageSource()); + def result = parser(doc, url) + if (result) { + rets << result + println rets.size() + } + if (rets.size() % 100 == 0) { + def add= saveData(rets) + println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")+" add:"+add + rets = [] + } + } + if(rets.size()>0){ + saveData(rets) + print "完" + } + } + + def parser(doc, url){ + def result = [:] + if(doc){ + def title = JQueryUtils.selectText("#prodnameb", doc) + def id = url.substring(url.indexOf("="), url.length()) + if(title){ + + def cityStr = JQueryUtils.selectText("dl:contains(所在城市:) li", doc) + if(cityStr && cityStr.contains("...")){ + def cityEles = JQueryUtils.select("dl:contains(所在城市:) li", doc) + if(cityEles){ + cityStr = cityEles[0].attr("title") + } + } + if(cityStr && cityStr.contains("省")){ + result.province = cityStr.substring(0, cityStr.indexOf("省")) + result.city = cityStr.substring(cityStr.indexOf("省")+1, cityStr.length()) + }else{ + if(cityStr.contains("自治区")){ + result.province = cityStr.substring(0, cityStr.indexOf("自治区")+3) + result.city = cityStr.substring(cityStr.indexOf("自治区")+3, cityStr.length()) + }else{ + result.city = cityStr.substring(0, cityStr.indexOf("市")) + } + } + + if(!result.city){ + return null + } + result.title = title + def rMap = OpenApiUtils.matchDistrict(result.city, result.title, null, true) + if (rMap) { + result.matchCity = rMap.city + result.matchRegion = rMap.region + result.matchName = rMap.name + result.matchAddress = rMap.address + result.houseNum = rMap.house_num + result.roomNum = rMap.room_num + } + + result.status = JQueryUtils.selectText("#message", doc).replace("本场竞买","") + + def t1 = JQueryUtils.selectText("#pricep", doc) + if(t1 && t1.contains("原因")){ + result.statusDesc = t1.substring(t1.indexOf("原因")+3, t1.toString().length()) + } + result.type = JQueryUtils.selectText("dl:contains(类  型:) dd", doc) + def enrollCount = JQueryUtils.selectText(["#signUpNum"], doc) + if(enrollCount) { + result.enrollCount = enrollCount.toInteger() + } + + def lookCount = JQueryUtils.selectText(["#visitorCount"], doc) + if(lookCount) { + result.lookCount = lookCount.toInteger() + } + + def callCount = JQueryUtils.selectText(["#remindPPNum"], doc) + if(callCount) { + result.callCount = callCount.toInteger() + } + + def curPriceStr = JQueryUtils.selectText(["#new_price_key"], doc) + if(curPriceStr) { + curPriceStr = curPriceStr.replace(",", "").replace("元", "").trim() + if(curPriceStr.isInteger()) { + result.curPrice = curPriceStr.toInteger() + }else { + result.curPrice = (int)(curPriceStr.toDouble()) + } + } + + def startPriceEles = JQueryUtils.select(["dl:contains(起始价格:) dd"], doc) + if(startPriceEles) { + def startPriceStr = startPriceEles[0].attr("title") + if(startPriceStr){ + startPriceStr = startPriceStr.replace(",", "").replace("¥","") + + if(startPriceStr.isInteger()) { + result.startPrice = startPriceStr.toInteger() + }else { + result.startPrice = (int)(startPriceStr.toDouble()) + } + } + + } + + + def addPriceEles = JQueryUtils.select(["dl:contains(加价幅度:) dd"], doc) + if(addPriceEles){ + def addPriceStr = addPriceEles[0].attr("title") + if(addPriceStr){ + addPriceStr = addPriceStr.replace(",", "").replace("¥","") + if(addPriceStr) { + result.addPrice = addPriceStr.toDouble().toInteger() + } + } + } + + def evaluationPriceEles = JQueryUtils.select(["dl:contains(评估价格:) dd"], doc) + if(evaluationPriceEles){ + def evaluationPriceStr = evaluationPriceEles[0].attr("title") + if(evaluationPriceStr){ + evaluationPriceStr = evaluationPriceStr.replace(",", "").replace("¥","") + if(evaluationPriceStr.isInteger()) { + result.evaluationPrice = evaluationPriceStr.toDouble().toInteger() + }else { + result.evaluationPrice = (int)(evaluationPriceStr.toDouble()) + } + } + } + + def depositPriceEles = JQueryUtils.select(["dl:contains(加价幅度:) dd"], doc) + if(depositPriceEles){ + def v = depositPriceEles[0].attr("title") + if(v){ + v = v.replace(",", "").replace("¥","") + if(v.isInteger()) { + result.depositPrice = v.toDouble().toInteger() + }else { + result.depositPrice = (int)(v.toDouble()) + } + } + } + + result.delayWeekly = JQueryUtils.selectText(["dl:contains(延时周期:) dd div:eq(0)"], doc) + result.person = JQueryUtils.selectText(["dl:contains(联系人:) dd"], doc) + result.court = JQueryUtils.selectText("#goToCourt", doc) + def mobileEles = JQueryUtils.select("dl:contains(联系电话:) dd", doc) + if(mobileEles){ + result.mobile = mobileEles[0].attr("title") + } + + def s1 = JQueryUtils.selectText("#div_announcement_gfword", doc) + if(s1) { + def start = s1.indexOf("法院将于") + def end = s1.indexOf("止") + if(start>-1 && end>-1 && end>start){ + s1 = s1.substring(s1.indexOf("法院将于"), s1.indexOf("止")) + } + def dateRangs = PaimaiDataUtil.getDateRange(s1) + if(dateRangs && dateRangs.size()==2) { + result.startDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[0], "yyyy年MM月dd日HH时") + result.finishDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[1], "yyyy年MM月dd日HH时") + } + } + + def trs = JQueryUtils.select([".rygtable table tr"], doc) + if(trs) { + def detailMap = [:] + for(def tr : trs) { + def d_tds = tr.select("td") + if(d_tds && d_tds.size()>=2) { + if(d_tds.size()==3) { + if(d_tds[2].text() && d_tds[2].text()!="/"){ + detailMap.put(d_tds[1].text(), d_tds[2].text()) + } + + }else { + if(d_tds[1].text() && d_tds[1].text()!="/") { + detailMap.put(d_tds[0].text(), d_tds[1].text()) + } + } + } + } + if(detailMap) { + result.detailMap = DataUtil.filterChar(detailMap) + } + if(!result.address) { + result.address = detailMap.get("拍品名称") + if(!result.address) { + result.address = detailMap.get("标的名称") + } + } + + } + + def imgs =JQueryUtils.select("#scroll li img", doc) + if(imgs) { + def imgLinks = [] + for(def img : imgs) { + def href = img.attr("src") + if(href.contains("icbc.com")) { + imgLinks << "https:"+href + } + } + result.imgUrls = imgLinks + } + + + def detailStr = JQueryUtils.selectText(".detail_list", doc) + if(detailStr){ + def area = DataUtil.getAreaNumber(detailStr, 5000) + if(area) { + result.area = area.toDouble() + if(result.tradePrice) { + result.tradeAvgPrice = (int)(result.tradePrice/area.toDouble()) + } + } + } + + result.content = com.fangjia.core.util.DataUtil.zip(doc.toString()) + result.introduce = detailStr.toString() + result.url = url + result.mDate = new Date() + result.source = "融e够" + return result; + } + + } + return null + } + + def getCallCount(id){ + def url= "https://gf.trade.icbc.com.cn/wholeSaleProd/visitorCount.jhtml?prodId="+id + def s = ABuYunUtil.doPost(url, null, null, "utf-8", true) + if(s){ + JSONObject jsonObject = JSONObject.fromObject(s); + println jsonObject + } + } + + + + + def saveData(def results){ + def add = 0 + for(def ret : results){ + def q = new BasicDBObject() + q.put("url", ret.url) + def dbo = paimaiResultColl.findOne(q); + + DBObject obj = new BasicDBObject(); + ret.each{k,v-> + obj.put(k,v) + } + + if(dbo){ + obj.m_date = new Date() + paimaiResultColl.update(q, new BasicDBObject('$set', obj)) + }else{ + add++ + obj.c_date = new Date() + obj.m_date = new Date() + try{ + paimaiResultColl.save(obj) + }catch(Exception e){ + println obj.url + } + + } + } + return add + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy index aac9c7315..3ea9f1d4a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy @@ -39,8 +39,6 @@ class JdPaimaiCrawl { println "共:"+ids.size() parserList( existsIds,ids, driver) - - } def getList(driver){ @@ -478,6 +476,13 @@ class JdPaimaiCrawl { } } + if(result.finishDate){ + result.finishDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.finishDate, "2000-01-01") + } + if(result.startDate){ + result.startDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.startDate, "2050-01-01") + } + result.content = com.fangjia.core.util.DataUtil.zip(doc.toString()) result.introduce = JQueryUtils.select([".floor-content div:eq(2) table"], doc).toString() result.url = url diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index f1750cfa9..6f99480ab 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -37,7 +37,7 @@ public class LianjiaCrawlByChrome { task.run() // task.run1() task.addDealRegion(); -// task.standardRegion() + task.standardRegion() } def run1(){ @@ -227,6 +227,10 @@ public class LianjiaCrawlByChrome { break; } } + if(page>=100){ + stop = true; + break; + } page++; }else{ stop = true; @@ -416,10 +420,11 @@ public class LianjiaCrawlByChrome { def cityDoc = Jsoup.parse(driver.getPageSource()); def cityEles = JQueryUtils.select(".city_province a", cityDoc) def cityMap = [:] + def citys= ["长春","吉林"] if(cityEles){ for(def e : cityEles) { def href = e.attr("href") - if (!href.contains("fang.lianjia") && e.text().contains("福州")) { + if (!href.contains("fang.lianjia") && citys.contains(e.text())){ cityMap.put(e.text(), href) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy index acced0959..c7451c5af 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy @@ -4,9 +4,12 @@ import com.alibaba.fastjson.JSON import com.fangjia.core.util.DataUtil import com.fangjia.script.crawl.JQueryUtils import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject +import com.mongodb.Bytes import com.mongodb.DBCollection import com.mongodb.DBObject +import com.rabbitmq.client.impl.AMQImpl import org.jsoup.Jsoup import org.openqa.selenium.WebDriver import org.openqa.selenium.chrome.ChromeDriver @@ -19,9 +22,91 @@ class PaimaiTest { DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai") + public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + public static final DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl) + + static main(args) { def task = new PaimaiTest(); - task.run(); +// task.run(); +// task.stat() +// task.updatePaimai() + task.updateFieldPaimai() + } + + def updateFieldPaimai(){ + def q = new BasicDBObject() + q.put("city", new BasicDBObject('$in',["重庆","重庆市"])) + def index = 0 + def dbcursor = paimai43_COLL.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + for(def obj : dbcursor) { + index++ + println index + if(obj.depositPrice){ + obj.put("depositPrice", obj.depositPrice.toInteger()) + paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj)) + } + } + } + + def updatePaimai(){ + def q = new BasicDBObject() + def dbcursor = paimaiResultColl.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def index =0 + for(def d : dbcursor){ + index++ + println index + def query = new BasicDBObject() + query.put("url", d.url) + def dbo = paimai43_COLL.findOne(query); + + DBObject obj = new BasicDBObject(); + d.each{k,v-> + obj.put(k,v) + } + + if(dbo){ + obj.m_date = new Date() + paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj)) + }else{ + obj.c_date = new Date() + obj.m_date = new Date() + try{ + paimai43_COLL.save(obj) + }catch(Exception e){ + println obj.url + } + } + } + } + + def stat(){ + def q = new BasicDBObject() + q.put("city", new BasicDBObject('$in', ["重庆","重庆市"])) + def dbcursor = paimaiResultColl.find(q) + def keys = [] + for(def db : dbcursor){ + if(db.matchCity){ + def key = db.matchCity +"##"+db.matchRegion+"##"+db.matchName; + if(!keys.contains(key)){ + keys << key + } +// try{ +// def jsonObject = OpenApiClient.addressMatch("重庆", db.address, "property", false, null, null) +// if(jsonObject != null && jsonObject.precise!=-1) { +// db.matchCity = jsonObject.get("city") +// db.matchRegion = jsonObject.get("district") +// db.matchName = jsonObject.get("name") +// paimaiResultColl.update(new BasicDBObject("_id", db._id),new BasicDBObject('$set',db)) +// println db.address+" "+db.matchName +// } +// }catch(Exception e){ +// println "error "+db.address +// } + + } + } + println keys.size() } def run(){ System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/TaobaoPaimaiCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/TaobaoPaimaiCrawl.groovy index f0b328f11..68b73d58e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/TaobaoPaimaiCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/TaobaoPaimaiCrawl.groovy @@ -1,7 +1,12 @@ package com.fangjia.script.gj.paimai import com.fangjia.script.crawl.JQueryUtils -import com.fangjia.script.thirdPriceCrawl.CrawalUtil +import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.util.DateUtils +import org.jsoup.Jsoup +import org.openqa.selenium.By +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.ChromeDriver class TaobaoPaimaiCrawl { static main(args) { @@ -9,55 +14,432 @@ class TaobaoPaimaiCrawl { task.run(); } def run(){ - def seedUrl = "https://sf.taobao.com/item_list.htm?spm=a213w.7398504.filter.86.675028eabFDx9r&category=50025969&auction_source=0&city=%C8%FD%D1%C7&sorder=2&st_param=-1&auction_start_seg=-1" - def regionUrls = regionCrawl(seedUrl); - def pageUrls = pageUrlCrawl(regionUrls) - detailCrawl(pageUrls) + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); +// def url = "https://sf.taobao.com/list/50025969__-1.htm?spm=a213w.7398504.filter.62.73e954b3fMWNLd&auction_source=0&st_param=-1&auction_start_seg=-1" + def url = "https://sf.taobao.com/list/50025969__2.htm?spm=a213w.7398504.filter.65.101554b3Nynwwp&auction_source=0&st_param=-1&auction_start_seg=-1" + WebDriver driver = new ChromeDriver() + getAllCityRegions(driver, url); } - def regionCrawl(def seedUrl){ - def document = CrawalUtil.getDocumentByABuYun(seedUrl, "", "gbk"); - def regionUrls = [] - if(document){ - def eles = JQueryUtils.select(".small-subcondion a", document) - if(eles){ - for(def el : eles){ - def reiongUrl = el.attr("href").trim() - if(reiongUrl.startsWith("//")){ - regionUrls << "http:"+reiongUrl + def getAllCityRegions(driver, url){ + + def provinces = getAllProvince(driver, url) + def citys = ["天津","上海","北京","重庆"] + + for(def province : provinces){ + driver.findElement(By.xpath("//a[contains(text(), '${province}')]")).click() + def doc = Jsoup.parse(driver.getPageSource()); + def cityEles= JQueryUtils.select(".unfold .J_FilterCity a") + for(def cityEle : cityEles){ + def city = cityEle.text() + if(citys.contains(city)){ + driver.findElement(By.xpath("//a[contains(text(), '${province}')]")).click() + doc = Jsoup.parse(driver.getPageSource()); + + driver.findElement(By.cssSelector(".J_FilterCity a")).click() + doc = Jsoup.parse(driver.getPageSource()); + def regionEles= JQueryUtils.select(".J_SubCondition .small-subcondio ul li a") + for(def r : regionEles){ + def m = [:] + m.province = province + m.city = city + m.region = r.text() + def urls = crawlLinks(m, driver) + parse(m, driver, urls) } + }else{ + def m = [:] + m.province = province + m.city = city + def urls = crawlLinks(m, driver) + parse(m, driver, urls) } } } - return regionUrls; } - def pageUrlCrawl(regionUrls){ + def crawlLinks(def m, def driver){ + def stop = false def urls = [] - for(def url : regionUrls){ - def document = CrawalUtil.getDocumentByABuYun(url, "", "gbk"); - if(document){ - def pages = JQueryUtils.selectText(".page-total",document) - if(pages){ - pages = pages.toInteger() - for(def page=1;page -1) { + result.address = addr.substring(0, addr.indexOf("室") + 1) + } else { + result.address = addr + } + + if (result.city) { + if (result.address.contains(city + "市")) { + result.address = result.address.substring(result.address.indexOf(city) + city.length() + 1) + } else { + result.address = result.address.substring(result.address.indexOf(city) + city.length()) + } + } + + + if (result.address.indexOf("】") > -1) { + result.address = result.address.substring(result.address.indexOf("】") + 1) + } + + + if (result.city) { + def rMap = OpenApiUtils.matchDistrict(result.city, result.title, null, true) + if (rMap) { + result.matchCity = rMap.city + result.matchRegion = rMap.region + result.matchName = rMap.name + result.matchAddress = rMap.address + result.houseNum = rMap.house_num + result.roomNum = rMap.room_num + } + } + } + if (statusDesc) { + def status = statusDesc.substring(statusDesc.indexOf("本场拍卖") + 5, statusDesc.length() - 1) + result.status = DataUtil.getPaiMaiType(status) + result.statusDesc = statusDesc + } + + def statusEle = JQueryUtils.select(".status-tip", doc) + if (statusEle) { + def sEle = JQueryUtils.select(".status-ing", doc) + if (sEle) { + result.status = "进行中" + } else { + result.status = "即将开始" + } + } + def curPrice = JQueryUtils.selectText(["#sf-price"], doc) + if (curPrice) { + if (curPrice.contains("当前价")) { + curPrice = JQueryUtils.selectText(["#sf-price .J_Price em:eq(0)"], doc) + if (curPrice) { + curPrice = curPrice.replace(",", "") + if (curPrice.isInteger()) { + result.curPrice = curPrice.toInteger() + } else { + result.curPrice = (int) (curPrice.toDouble()) + } + } + } else if (curPrice.contains("成交价")) { + def tradePrice = JQueryUtils.selectText(["#sf-price .J_Price em:eq(0)"], doc) + if (tradePrice) { + tradePrice = tradePrice.replace(",", "") + if (tradePrice.isInteger()) { + result.tradePrice = tradePrice.toInteger() + } else { + result.tradePrice = (int) (tradePrice.toDouble()) + } + + } + } else if (curPrice.contains("起拍价")) { + def startPrice = JQueryUtils.selectText(["#sf-price .J_Price em:eq(0)"], doc) + if (startPrice) { + startPrice = startPrice.replace(",", "") + if (startPrice.isInteger()) { + result.startPrice = startPrice.toInteger() + } else { + result.startPrice = (int) (startPrice.toDouble()) + } + + } + } + } + + def finishStr = JQueryUtils.selectText([".pm-bid-eyebrow .over-title"], doc) + if (finishStr) { + if (finishStr.contains("结束时间")) { + finishStr = JQueryUtils.select([".pm-main .J_TimeLeft"], doc).text() + if (finishStr) { + result.finishDate = DateUtils.parseFromTightByPattern(finishStr, "yyyy/MM/dd HH:mm:ss") + } + } else if (finishStr.contains("成交时间")) { + finishStr = JQueryUtils.select([".pm-main .J_TimeLeft"], doc).text() + if (finishStr) { + result.finishDate = DateUtils.parseFromTightByPattern(finishStr, "yyyy/MM/dd HH:mm:ss") + } + } + } + + def enrollCount = JQueryUtils.selectText([".pm-remind .J_Applyer"], doc) + if (enrollCount) { + result.enrollCount = enrollCount.toInteger() + } + + def callCount = JQueryUtils.selectText(["#J_NotifyNum"], doc) + if (callCount) { + result.callCount = callCount.toInteger() + } + + def lookCount = JQueryUtils.selectText(["#J_Looker"], doc) + if (lookCount) { + result.lookCount = lookCount.toInteger() + } + def tds = JQueryUtils.select(["#J_HoverShow td"], doc) + if (tds && tds.size() > 0) { + for (def td : tds) { + def str = JQueryUtils.selectText("span:eq(0)", td).replace(" ", "") + if (str) { + if (str.contains("变卖价")) { + def str1 = td.select(".J_Price").text() + def str2 = td.select(".J_Price .J_Price-tip").text() + def p = str1.replace(str2, "").replace(",", "") + if (p) { + if (p.isInteger()) { + result.sellOffPrice = p.toInteger() + } else { + result.sellOffPrice = (int) (p.toDouble()) + } + + } + } else if (str.contains("加价幅度")) { + def str1 = td.select(".J_Price").text() + def str2 = td.select(".J_Price .J_Price-tip").text() + def p = str1.replace(str2, "").replace(",", "") + if (p) { + if (p.isInteger()) { + result.addPrice = p.toInteger() + } else { + result.addPrice = (int) (p.toDouble()) + } + + } + } else if (str.contains("类型")) { + + result.type = td.select(".pay-type").text() + + } else if (str.contains("保证金")) { + def str1 = td.select(".J_Price").text() + def str2 = td.select(".J_Price .J_Price-tip").text() + def p = str1.replace(str2, "").replace(",", "") + if (p) { + if (p.isInteger()) { + result.depositPrice = p.toInteger() + } else { + result.depositPrice = (int) (p.toDouble()) + } + } + } else if (str.contains("变卖周期")) { + + result.sellOffWeekly = td.select("span:eq(1)").text().replace(": ", "") + + } else if (str.contains("竞价周期")) { + + result.pricePkWeekly = td.select("span:eq(1)").text().replace(": ", "") + + } else if (str.contains("优先购买权")) { + + result.preemptivePerson = td.select("span:eq(1)").text().replace(": ", "") + + } else if (str.contains("变卖预缴款")) { + def str1 = td.select("span:eq(1) .J_Price").text() + def str2 = td.select("span:eq(1) .J_Price .J_Price-tip").text() + def p = str1.replace(str2, "").replace(",", "") + if (p) { + if (p.isInteger()) { + result.sellOffAdvancePay = p.toInteger() + } else { + result.sellOffAdvancePay = (int) (p.toDouble()) + } + } + + } else if (str.contains("评估价")) { + def str1 = td.select(".J_Price").text() + def str2 = td.select(".J_Price .J_Price-tip").text() + def p = str1.replace(str2, "").replace(",", "") + if (p) { + if (p.isInteger()) { + result.evaluationPrice = p.toInteger() + } else { + result.evaluationPrice = (int) (p.toDouble()) + } + } + } else if (str.contains("延时周期")) { + + result.delayWeekly = td.select("span:eq(1)").text().replace(": ", "") + + } else if (str.contains("起拍价")) { + def str1 = td.select(".J_Price").text() + def str2 = td.select(".J_Price .J_Price-tip").text() + def p = str1.replace(str2, "").replace(",", "") + if (p) { + if (p.isInteger()) { + result.startPrice = p.toInteger() + } else { + result.startPrice = (int) (p.toDouble()) + } + } + } + } + } + } + + result.rule = JQueryUtils.selectText(".pai-info .pai-content", doc) + + def trs = JQueryUtils.select(["#J_desc table tr"], doc) + if (trs) { + def detailMap = [:] + for (def tr : trs) { + def d_tds = tr.select("td") + if (d_tds && d_tds.size() >= 2) { + if (d_tds.size() == 3) { + detailMap.put(d_tds[1].text(), d_tds[2].text()) + } else { + detailMap.put(d_tds[0].text(), d_tds[1].text()) + } + } else { + if (d_tds[0] && d_tds[0].text().contains("拍品介绍")) { + detailMap.put("拍品介绍", d_tds[0].text().replace("拍品介绍 ", "")) + } + } + } + + if (detailMap) { + result.detailMap = DataUtil.filterChar(detailMap) + } + + if (!result.address) { + result.address = detailMap.get("拍品名称") + if (!result.address) { + result.address = detailMap.get("标的名称") + } + } + + } + + def imgs = JQueryUtils.select([".slide-bigpic img"], doc) + if (imgs) { + def imgLinks = [] + for (def img : imgs) { + def href = img.attr("src") + if (href.contains("http")) { + imgLinks << href + } else { + imgLinks << url.substring(0, url.indexOf(":") + 1) + href + } + + } + result.imgUrls = imgLinks + } + } + + //面积 + def article = JQueryUtils.selectText(["#J_ItemNotice"], doc) + if (article) { + def area = DataUtil.getAreaNumber(article, 5000) + if (!area) { + article = JQueryUtils.selectText(["#J_NoticeDetail"], doc) + if (article) { + area = DataUtil.getAreaNumber(article, 5000) + } + } + if (!area) { + article = JQueryUtils.selectText(["body"], doc) + if (article) { + area = DataUtil.getAreaNumber(article, 5000) + } + } + if (area) { + result.area = area.toDouble() + if (result.tradePrice) { + result.tradeAvgPrice = (int) (result.tradePrice / area.toDouble()) } } } - } - return urls; - } - def detailCrawl(pageUrls){ - if(pageUrls){ - for(def pageUrl : pageUrls){ - def document = CrawalUtil.getDocumentByABuYun(pageUrl, "", "gbk"); - if(document){ - def pages = JQueryUtils.select(".page-total", document) + + def s1 = JQueryUtils.selectText(["#J_NoticeDetail"], doc) + if (s1) { + def dateRangs = DataUtil.getDateRange(s1) + if (dateRangs) { + for (def j = 0; j < dateRangs.size(); j++) { + def date = dateRangs[j] + def pattern = null + if (date.endsWith("日时") || date.endsWith("日")) { + pattern = "yyyy年MM月dd日" + } else { + + def ind = date.indexOf("日") + if (date.indexOf("时") == -1 && date.toString().length() > ind + 1) { + date = date + "时" + } + + pattern = "yyyy年MM月dd日HH时" + } + if (j == 0) { + result.startDate = DateUtils.parseFromTightByPattern(date, pattern) + } else { + if (!result.finishDate) { + result.finishDate = DateUtils.parseFromTightByPattern(date, pattern) + } + } + } + } + + if (!result.startPrice) { + result.startPrice = DataUtil.getStartPrice(s1) } } + + result.content = com.fangjia.core.util.DataUtil.zip(doc.toString()) + def estateType = DataUtil.findEstateType(s) + if (estateType) { + result.matchEstateType = estateType + } + + result.introduce = JQueryUtils.select(["#J_desc table"], doc).toString() + result.url = url + result.mDate = new Date() + results << result } + return results; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcChromeCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcChromeCrawl.groovy index e5ffbd54f..172cc24b5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcChromeCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcChromeCrawl.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.loudong +import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.crawl.JQueryUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject @@ -16,8 +17,15 @@ class CqfdcChromeCrawl { static main(args) { - def task = new CqfdcChromeCrawl(); - task.run() +// def task = new CqfdcChromeCrawl(); +// task.run() + + def name = "E3号楼" //69幢(规划C-7号楼) 规划5-5号楼 + def s = DataHandleUtils.findMatchPattern(name, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?") + println s + + def unitNumStr = DataHandleUtils.findMatchPattern(name, "\\d+单元") + println unitNumStr } def run(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy index 1ce1c48c7..9fbafb2ae 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy @@ -1,20 +1,49 @@ package com.fangjia.script.loudong +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.linxsh.util.AddressUtil import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject +import com.mongodb.Bytes import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Matcher +import java.util.regex.Pattern class CqfdcMatch { + public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", lineUrl) + static DBCollection loudongFdcConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong_cqfdc", "192.168.0.235", 27018) static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong", "192.168.0.235", 27018) + + public static final DBCollection LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia_base", "loudong", lineUrl) + public static final DBCollection HOUSE_LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", lineUrl) + + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + static main(args) { def task = new CqfdcMatch(); // task.run() // task.stat() - task.export2() +// task.export2() +// task.matchLoudong() + + task.addLoudong() +// task.addFdcLoudong() } def export(){ def q = new BasicDBObject() @@ -139,6 +168,378 @@ class CqfdcMatch { return rets; } + def addHouseLoudong(){ + def q = new BasicDBObject() + def dbcursor = loudongConn.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def keys = [] + def index = 0 + for(def d : dbcursor) { + index++ + println index + if(d.room_num && (d.room_num.contains("商") || d.room_num.contains("社区用房"))){ + continue + } + if(d.area.toDouble()<=25){ + continue + } + def newRegion = RegionData.formatRegion(d.city, d.region) + if (newRegion) { + def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?") + def house_num = null + if (houseNumStr) { + house_num = houseNumStr.replace("号楼", "").replace("幢", "").replace("栋", "").replace("号", "") + } + if (house_num) { + def unitNum = DataHandleUtils.findMatchPattern(d.house_num, "\\d+单元") + if (unitNum) { + unitNum = unitNum.replace("单元", "") + } + def key = d.city + "##" + newRegion + "##" + d.district_name + "##" + house_num+ "##" + unitNum +"##"+d.room_num + if(!keys.contains(key) && d.room_num) { + keys << key + def count = findHouseLoudongDatas(d.city, newRegion, d.district_name, house_num, unitNum, d.room_num) + if(!count){ + def dist = null + if(d.match_city){ + dist = CommonDatautils.findDistrict(d.match_city, d.match_region, d.match_name,"district") + } + def udata = new BasicDBObject() + udata.put("city", d.city) + udata.put("region", newRegion) + udata.put("name", d.district_name) + udata.put("house_num", house_num) + if(unitNum){ + udata.put("house_num_unit", unitNum) + } + udata.put("room_num", d.room_num) + udata.put("house_area",d.area.toDouble()) + udata.put("status", 0) + udata.put("from", "房谱网") + if(d.floor_heigth && d.floor_heigth.isNumber()){ + udata.put("house_height", d.floor_heigth.toInteger()) + } + if(d.floor && d.floor.isNumber()){ + udata.put("house_floor", d.floor.toInteger()) + } + if(dist){ + udata.put("district_id", dist._id.toString()) + udata.put("name_in_seaweed", dist.name) + } + if(d.address){ + def address = AddressUtil.formatAddress(d.city, d.address) + if(address){ + udata.put("address", address) + } + } + udata.put("c_date", new Date()) + udata.put("m_date", new Date()) + HOUSE_LOUDONG_LINE_COLL.save(udata) + } + } + } + } + } + } + + def findHouseLoudongDatas(city, region, district_name, house_num, unitNum, room_num){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", district_name) + q.put("house_num", house_num) + if(unitNum){ + q.put("unit_num", unitNum) + } + q.put("room_num", room_num) + def count = HOUSE_LOUDONG_LINE_COLL.count(q) + return count; + } + + def findAllLoundongByHouseNum(){ + def q = new BasicDBObject() + def dbcursor = loudongConn.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(5000) + def keys = [] + for(def d : dbcursor){ + def newRegion = RegionData.formatRegion(d.city, d.region) + if(newRegion){ + def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?") + def house_num = null + if(houseNumStr){ + house_num = houseNumStr.replace("号楼","").replace("幢","").replace("栋","").replace("号","") + } + if(house_num){ + def unitNum = DataHandleUtils.findMatchPattern(d.house_num, "\\d+单元") + if(unitNum){ + unitNum = unitNum.replace("单元","") + } + def key = d.city +"##"+newRegion+"##"+d.district_name+"##"+house_num + if(unitNum){ + key = key +"##"+unitNum + } + if(!keys.contains(key)){ + keys << key + def obj = findLoudongOnLine(d.city, newRegion, d.district_name, house_num, d.unit_num) + println d.district_name+" "+d.house_num+"==${house_num}==${unitNum}" + if(obj){ + def udata = new BasicDBObject() + def areas = obj.areas + if(!areas.contains(d.area)){ + areas << d.area + + def forms = obj.from + if(!forms.contains("房谱网")){ + forms << "房谱网" + } + obj.m_date = new Date() + def query = new BasicDBObject("_id", obj._id) + LOUDONG_LINE_COLL.update(query, new BasicDBObject('$set', obj)) + + } + }else{ + def udata = new BasicDBObject() + udata.put("city", d.city) + udata.put("region", newRegion) + udata.put("name", d.district_name) + udata.put("house_num", house_num) + if(unitNum){ + udata.put("unit_num", unitNum) + } + if(d.address){ + def address = AddressUtil.formatAddress(d.city, d.address) + if(address){ + udata.put("address", address) + } + } + udata.put("areas",[d.area]) + udata.put("from",["房谱网"]) + if(d.floor_heigth && d.floor_heigth.toString().toInteger()){ + udata.put("floors", d.floor_heigth.toString().toInteger()) + } + + udata.put("c_date", new Date()) + udata.put("status", 1) + LOUDONG_LINE_COLL.save(udata) + } + } + } + + } + } + } + + def addFdcHouseLoudong(){ + def q = new BasicDBObject() + def dbcursor = loudongFdcConn.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(2000) + def keys = [] + def index = 0 + for(def d : dbcursor){ + index++ + println index + if(!d.use_type){ + continue + } + if(!d.use_type.contains("住宅")){ + continue + } + def newRegion = d.match_region + if(newRegion){ + def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num_desc, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?") + def house_num = null + if(houseNumStr){ + house_num = houseNumStr.replace("号楼","").replace("幢","").replace("栋","").replace("号","") + } + if(house_num){ + def key = d.city +"##"+newRegion+"##"+d.name+"##"+house_num+"##"+d.unit_num+"##"+d.room_num + if(!keys.contains(key)) { + keys << key + def count = findHouseLoudongDatas(d.city, newRegion, d.name, house_num, d.unit_num, d.room_num) + if(!count){ + def maxHeight = findMaxHeight(d.city, d.name, d.house_num_desc, d.unit_num) + def dist = null + if(d.match_city){ + dist = CommonDatautils.findDistrict(d.match_city, d.match_region, d.match_name,"district") + } + def floor = null + if(d.room_num && d.room_num.contains("-")) { + def arrs = d.room_num.split("-") + floor = arrs[0] + } + Integer room = null + Integer hall = null + if(d.room_type && d.room_type.contains("室")){ + def roomStr = d.room_type.substring(0,d.room_type.indexOf("室")) + if(roomStr){ + roomStr = roomStr.replace("一","1").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("六","6").replace("七","7").replace("八","8").replace("九","9").replace("十","10") + room = roomStr.toInteger() + } + + if(d.room_type.contains("厅")){ + def hallStr = d.room_type.substring(d.room_type.indexOf("室")+1, d.room_type.indexOf("厅")) + if(hallStr){ + hallStr = hallStr.replace("一","1").replace("两","2").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("六","6").replace("七","7").replace("八","8").replace("九","9").replace("十","10") + hall = hallStr.toInteger() + } + } + } + + + def udata = new BasicDBObject() + udata.put("city", d.city) + udata.put("region", newRegion) + udata.put("name", d.name) + udata.put("house_num", house_num) + if(d.unit_num){ + udata.put("house_num_unit", d.unit_num) + } + udata.put("room_num", d.room_num) + udata.put("house_area",d.area.toDouble()) + udata.put("status", 0) + udata.put("from", "重庆网上房地产") + udata.put("house_height", maxHeight) + if(room){ + udata.put("room", room) + } + if(hall){ + udata.put("hall", hall) + } + if(floor && floor.isNumber()){ + udata.put("house_floor", floor.toInteger()) + } + if(dist){ + udata.put("district_id", dist._id.toString()) + udata.put("name_in_seaweed", dist.name) + } + if(d.address){ + def address = AddressUtil.formatAddress(d.city, d.address) + if(address){ + udata.put("address", address) + } + } + udata.put("c_date", new Date()) + udata.put("m_date", new Date()) + HOUSE_LOUDONG_LINE_COLL.save(udata) + } + } + } + } + } + } + + public String getMatchString(String text,String pattern){ + Pattern p=Pattern.compile(pattern); + Matcher m=p.matcher(text); + if(m.find()){ + return m.group(0) + } + return null; + } + + def addFdcBaseLoudong(){ + def q = new BasicDBObject() + def dbcursor = loudongFdcConn.find(q).batchSize(5000) + def keys = [] + for(def d : dbcursor){ + if(!d.match_region){ + continue + } + def newRegion = d.match_region + + def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num_desc, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?") + def house_num = null + if(houseNumStr){ + house_num = houseNumStr.replace("号楼","").replace("幢","").replace("栋","").replace("号","") + } + if(house_num){ + def key = d.city +"##"+newRegion+"##"+d.name+"##"+house_num + if(d.unit_num) { + key = key + "##" + d.unit_num + + if (!keys.contains(key)) { + keys << key + def obj = findLoudongOnLine(d.city, newRegion, d.name, house_num, d.unit_num) + println d.name + " " + d.house_num_desc + "==${house_num}==${d.unit_num}" + + def maxHeight = findMaxHeight(d.city, d.name, d.house_num_desc, d.unit_num) + if (obj) { + + def areas = obj.areas + if (!areas.contains(d.area)) { + areas << d.area + + def forms = obj.from + if (!forms.contains("重庆网上房地产")) { + forms << "重庆网上房地产" + } + obj.m_date = new Date() + def query = new BasicDBObject("_id", obj._id) + LOUDONG_LINE_COLL.update(query, new BasicDBObject('$set', obj)) + + } + } else { + def udata = new BasicDBObject() + udata.put("city", d.city) + udata.put("region", newRegion) + udata.put("name", d.name) + udata.put("house_num", house_num) + if (d.unit_num) { + udata.put("unit_num", d.unit_num) + } + if (d.address) { + def address = AddressUtil.formatAddress(d.city, d.address) + if (address) { + udata.put("address", address) + } + } + udata.put("areas", [d.area]) + udata.put("from", ["重庆网上房地产"]) + udata.put("floors", maxHeight) + udata.put("c_date", new Date()) + udata.put("status", 1) + LOUDONG_LINE_COLL.save(udata) + } + } + } + } + } + } + + def findMaxHeight(city, name,house_num, unit_num){ + def q = new BasicDBObject() + q.put("city",city) + q.put("name",name) + q.put("house_num_desc",house_num) + q.put("unit_num",unit_num) + def dbcursor = loudongFdcConn.find(q) + def heights = [] + for(def h : dbcursor){ + if(h.nominal_floor && h.nominal_floor.isNumber()){ + def height = h.nominal_floor.toInteger() + if(!heights.contains(height)){ + heights << height + } + } + } + if(heights){ + heights = heights.sort() + return heights[heights.size()-1] + } + return null + } + + def findLoudongOnLine(city, region, name, house_num, unit_num){ + def q = new BasicDBObject() + q.put("city",city) + q.put("region",region) + q.put("name",name) + q.put("house_num",house_num) + if(unit_num){ + q.put("unit_num",unit_num) + } + def dbo = LOUDONG_LINE_COLL.findOne(q) + return dbo + } + + def stat(){ def q = new BasicDBObject() @@ -168,6 +569,135 @@ class CqfdcMatch { } + def matchLoudong(){ + def names = findAllLoudong("重庆") + print names.size() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(names.size()) + + def retMap = new ConcurrentHashMap() + for(def i=0;i + def arrs = key.split("#") + + def q = new BasicDBObject("city", arrs[0]) + q.put('region', arrs[1]); + q.put('name', arrs[2]); + + def udata = new BasicDBObject() + udata.put("district_id", m.id.toString()) + udata.put("name_in_seaweed", m.name) + udata.put("district_no", m.sid) + HOUSE_LOUDONG_COLL.update(q, new BasicDBObject('$set', udata), false, true); + index++ + println index + } + } + + def findAllLoudong(city){ + def q = new BasicDBObject() + q.put("city", city) + q.put("district_id",new BasicDBObject('$exists', false)) + + def keys = new BasicDBObject() + keys.put("city", 1) + keys.put("region", 1) + keys.put("name", 1) + def dbcursor = HOUSE_LOUDONG_COLL.find(q).batchSize(5000) + def names = new CopyOnWriteArrayList() + for(def db : dbcursor){ + def key = db.city +"#"+db.region+"#"+db.name + if(!names.contains(key)){ + names << key + } + } + return names; + } + + def addLoudong(){ +// def stat = statBaseLoudong() +// println stat +// findAllLoundongByHouseNum(); + def stat1 = statHouseLoudong() + println stat1 +// addHouseLoudong() + } + + def addFdcLoudong(){ +// addFdcBaseLoudong() + addFdcHouseLoudong() + } + + def statHouseLoudong(){ + def q = new BasicDBObject() + q.put("city", "重庆") + def dbcursor = HOUSE_LOUDONG_LINE_COLL.find(q) + def total = dbcursor.size() + def ids = [] + for(def d : dbcursor){ + if(d.district_id && !ids.contains(d.district_id)){ + ids << d.district_id + } + } + def map = [:] + map.rows_total = total + map.dist_count = ids.size() + return map + } + + def statBaseLoudong(){ + def q = new BasicDBObject() + q.put("city", "重庆") + def dbcursor = LOUDONG_LINE_COLL.find(q) + def total = dbcursor.size() + def ids = [] + for(def d : dbcursor){ + if(d.id_in_seaweed && !ids.contains(d.id_in_seaweed)){ + ids << d.id_in_seaweed + } + } + def map = [:] + map.rows_total = total + map.dist_count = ids.size() + return map + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy index 795b1556a..e308339de 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy @@ -12,7 +12,8 @@ import com.fangjia.script.util.DateUtils class DingTalkUtils { static main(String[] args) { - Date now = DateUtils.parseFromTightByPattern("2021-08-01", "yyyy-MM-dd") ; + // 要导出的月份第一天 + Date now = DateUtils.parseFromTightByPattern("2022-03-01", "yyyy-MM-dd") ; String startTime = DateUtils.addMonth(now, -1) + " 00:00:00" String monthStartTime = DateUtils.formatDateByPattern(now, "yyyy-MM-dd HH:mm:ss") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/geo/BlockRegion.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/geo/BlockRegion.groovy new file mode 100644 index 000000000..bdd0d1110 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/geo/BlockRegion.groovy @@ -0,0 +1,46 @@ +package com.fangjia.script.temp.shenjian.geo + +import com.fangjia.script.gj.location.Point +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.xh.InsidePolygon +import com.mongodb.BasicDBList +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "block", "192.168.0.85", 28017) + +collection.find(new BasicDBObject()).forEach({ obj -> + if (obj.type == 'block') { + if (obj.region instanceof String) { + return; + } + BasicDBList reigon = (BasicDBList)obj.region + int count = 0 + List info = new ArrayList<>() + for (String s : reigon) { + info.add(s) + } + if (info.size() > 1) { + boolean find = false + for (String s : info) { + if (find) { + break + } + collection.find(new BasicDBObject("type", "region").append("name", s)).limit(1).forEach({obj2 -> + String border = obj2.border + String points = border.replaceAll(",", "-").replaceAll(";", "-") + List pointList = InsidePolygon.createPoint(points); + com.fangjia.script.looplineinfo.Point point = new com.fangjia.script.looplineinfo.Point(obj.lng, obj.lat) + if (InsidePolygon.aboutZone(point, pointList)) { + collection.update(new BasicDBObject("_id", obj._id), + new BasicDBObject('$set', new BasicDBObject("region", obj2.name))); + find = true; + } + }) + } + } else { + collection.update(new BasicDBObject("_id", obj._id), + new BasicDBObject('$set', new BasicDBObject("region", info.get(0)))); + } + } +}) \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetLngLatFromRegionBlock.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetLngLatFromRegionBlock.groovy new file mode 100644 index 000000000..c23db19ed --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetLngLatFromRegionBlock.groovy @@ -0,0 +1,27 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang.StringUtils + +def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; +final DBCollection ZS_REGION_CONN = MongoUtils.getCollectionByName2("fangjia", "region_block", ZS_MONGODB_URL) + +ZS_REGION_CONN.find(new BasicDBObject()).batchSize(1000).each {obj -> + def city = obj.city + def org_name = obj.org_name + + List result = GaoDeUtils.searchPoi(city, org_name, org_name, null) + if (CollectionUtils.isNotEmpty(result)) { + def location = result.get(0).location + def lon = location.split(",")[0] + def lat = location.split(",")[1] + ZS_REGION_CONN.update(new BasicDBObject("_id", obj._id), + new BasicDBObject('$set',new BasicDBObject("location", new BasicDBObject("lon", lon).append("lat", lat)))) + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy index 668172385..c17285a78 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy @@ -128,10 +128,10 @@ class SeaweedUtils { */ static formatHouseholdCount(def household_count) { if (household_count && FieldValidatorUtils.checkValue(household_count) ) { - if (Long.valueOf(household_count) < 50) { + if (Integer.valueOf(household_count) < 50) { return "" // 户数 } - return household_count + return household_count.toInteger() } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index ee0edcfe2..c676385ed 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -40,13 +40,13 @@ class Esf58Crawl extends BaseBatch{ def static useDistKeys = [] static def totalCount = new AtomicLong() - def cityList = ["成都":"cd|102","杭州":"hz|79","重庆":"cq|37","苏州":"su|5","无锡":"wx|93", "深圳":"sz|4","郑州":"zz|342", "长沙":"cs|414","东莞":"dg|413", + def cityList = ["福州":"fz|304","成都":"cd|102","杭州":"hz|79","重庆":"cq|37","苏州":"su|5","无锡":"wx|93", "深圳":"sz|4","郑州":"zz|342", "长沙":"cs|414","东莞":"dg|413", "保定":"bd|424","保亭":"baoting|10367","南宁":"nn|845", "扬州":"yz|637","汕头":"st|783","泰州":"taizhou|693","淮安":"ha|968","湛江":"zhanjiang|791","锦州":"jinzhou|2354","昆明":"km|541","柳州":"liuzhou|7133","肇庆":"zq|901","连云港":"lyg|2049","日照":"rizhao|3177","常德":"changde|872", "沧州":"cangzhou|652", "泉州":"qz|291","清远":"qingyuan|7303", "西宁":"xn|2052","遵义":"zunyi|7620","株洲":"zhuzhou|1086","贵阳":"gy|2015","岳阳":"yy|821","徐州":"xz|471", "哈尔滨":"hrb|202","宜昌":"yc|858", "廊坊":"lf|772","秦皇岛":"qhd|1078","惠州":"huizhou|722","洛阳":"luoyang|556", "石家庄":"sjz|241","太原":"ty|740","盐城":"yancheng|613","信阳":"xy|8694","南充":"nanchong|2378","南阳":"ny|592","厦门":"xm|606", "绵阳":"mianyang|1057","济宁":"jining|450","邯郸":"hd|572", "淮北":"huaibei|9357","滁州":"chuzhou|10266", - "福州":"fz|304","蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158", + "蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158", "兰州":"lz|952","合肥":"hf|837", "乌鲁木齐":"xj|984","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","潍坊":"wf|362", "芜湖":"wuhu|2045","东莞":"dg|413","常州":"cz|463","佛山":"fs|222", "南通":"nt|394","吉林":"jl|700","中山":"zs|771","阜阳":"fy|2325", @@ -192,37 +192,37 @@ class Esf58Crawl extends BaseBatch{ void process(Object args) { //addContent("福州","https://fz.58.com/xiaoqu/", 70); - def citys = ["佛山","潍坊","长春","太原","南通","厦门","呼和浩特","常州","中山","德阳","镇江","泰安","银川","襄阳","江门","荆州","上海","衡阳","滨州","遂宁","鞍山","赣州","阜阳","枣庄","聊城","芜湖","吉林","德州","信阳","新乡","宿迁","沧州","桂林","包头","常德","乐山","遵义","眉山","株洲","驻马店","日照","许昌","成都","衡水","宜宾","泸州","营口","达州","宝鸡","广安","吉安","孝感","鄂尔多斯","十堰","清远","湘潭","上饶","漯河","邢台","平顶山","菏泽","宜春","承德","赤峰","延边","茂名","焦作","内江","韶关","梅州","郴州","黄冈","红河","绥化","娄底","丹东","邵阳","开封","阳江","黄石","周口","牡丹江","长治","咸宁","汉中","自贡","衢州","晋中","永州","北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏","重庆"] - for(def city : citys){ - def str = cityList.get(city) - for(def i=0;i<1;i++){ - def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); - if(districts){ - def arrs = str.split("\\|") - def url = "https://"+arrs[0]+".58.com/xiaoqu/" - distMap.put(city, districts) - addCrawlKeys(city); - addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); - } - } - } - -// cityList.each{city, str-> +// def citys = ["北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏"] +// for(def city : citys){ +// def str = cityList.get(city) // for(def i=0;i<1;i++){ -// if(citys.contains(city)){ -// def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); -// if(districts){ -// def arrs = str.split("\\|") -// def url = "https://"+arrs[0]+".58.com/xiaoqu/" -// distMap.put(city, districts) -// -// addCrawlKeys(city); -// addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); -// } +// def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); +// if(districts){ +// def arrs = str.split("\\|") +// def url = "https://"+arrs[0]+".58.com/xiaoqu/" +// distMap.put(city, districts) +// addCrawlKeys(city); +// addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); // } // } // } + cityList.each{city, str-> + for(def i=0;i<1;i++){ +// if(citys.contains(city)){ + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); + if(districts){ + def arrs = str.split("\\|") + def url = "https://"+arrs[0]+".58.com/xiaoqu/" + distMap.put(city, districts) + + addCrawlKeys(city); + addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); + } +// } + } + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 279f5f3c1..fec88b8c5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -18,7 +18,7 @@ class SynchDistrictPrice { // def sourceMap = ["fang.com":"搜房","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房", // "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"] //"fang.com":"搜房", - def sourceMap = ["ke.com":"贝壳找房","fang.com":"搜房","cityhouse.cn":"城市房产"] + def sourceMap = ["fang.com":"搜房","ke.com":"贝壳找房","5i5j.com":"我爱我家"] static main(args) { def task = new SynchDistrictPrice() task.run() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index bd0fbb3f3..4866f25b8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] for(def source :sources){ - def count1 = findTotalCount(source, 202202); - def count2 = findTotalCount(source, 202203); + def count1 = findTotalCount(source, 202203); + def count2 = findTotalCount(source, 202204); println source +" "+count1 +" "+count2 } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy index d37680fac..fe433f087 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy @@ -245,6 +245,48 @@ public class CommonUtils { } + + static void recalculationSqRent(String city, String region, String name, String estateType, Date calcDate = new Date(), boolean isTest = false, int calcType = 0, needRedrawTrend = true, def sync = false) { + + + calcDate = com.fangjia.core.util.DateUtil.truncateDay(calcDate) + if (estateType && estateType.contains("别墅")) { + estateType = "别墅" + } else if (estateType) { + estateType = "小区" + } + + DBObject cmd = new BasicDBObject() + cmd.put("city", city) + cmd.put("region", region) + cmd.put("name", name) + cmd.put('estate_type', estateType) + cmd.put("calc_date", calcDate) + cmd.put("calc_type", 0) + println(cmd) + DBObject data = new BasicDBObject() + data.put("category", "square_rent") + data.put("priority", 3) + data.put("status", 0) + data.put("remark", 'week') + data.put("c_date", new Date()) + + + if(isTest){ + data.put("ip", "192.168.0.231") + COMMAND_COLL.update(cmd, new BasicDBObject('$set', data), true, false) + }else{ + data.put("ip", ["fangjia-priceanalysis-0", "fangjia-priceanalysis-1"][new Random().nextInt(2)]) + COMMAND_COLL.update(cmd, new BasicDBObject('$set', data), true, false) + } + if(sync){ + while (COMMAND_COLL.count(cmd) != 0){ + Thread.sleep(1000) + } + } + + } + /** * 重算小区均价 * @param city diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy index c39e16b2e..7e6e73263 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy @@ -305,6 +305,43 @@ public class DbUtils{ } return districts + } + + /** + * 查询附近一共里内小区的数量 + */ + def static findAroundDistrictByRadius(BasicDBObject district, def radius){ + def districts = [] + if(hasGeo(district)){ + def query = new BasicDBObject() + query.append("cat", "district") + query.append("city", district.city) + query.append("visible", 0) + DBObject sdq = new BasicDBObject(); + def lat = district.lat / 10000000000.0 + def lng = district.lng / 10000000000.0 + sdq.put("sd.point", lat + "," + lng); + sdq.put("sd.sfield", "lat_lng"); + sdq.put("sd.distance", String.valueOf((double) radius / 1000.0)); + query.append('$sd', sdq) + + def keys = new BasicDBObject() + keys.append("city", 1) + keys.append("region", 1) + keys.append("name", 1) + keys.append("lat", 1) + keys.append("lng", 1) + keys.append("lat2", 1) + keys.append("lng2", 1) + keys.append("estate_type2", 1) + DbUtils.SEAWEED_SLAVE.find(query, keys).each {BasicDBObject dbo-> + if(!district._id || dbo._id != district._id){ + districts << dbo + } + } + } + return districts + } /** * 查询小区均价 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index 5ef52da25..29c5f5f4b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -229,6 +229,93 @@ public class OpenApiClient { return null } + public static JSONObject searchLoudong(String id, String houseNum, String unitNum){ + try{ + Map params = new HashMap(); + params.put("id", id); + params.put("houseNumber", houseNum); + params.put("unitNumber", unitNum); + params.put("token", TOKEN); + String content = new HttpRequest().get("http://localhost:8080/fangjia-open/property/searchLoudong", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + while (result == null) { + result = HttpRequestUtil.getJsonContent("http://localhost:8080/fangjia-open/property/searchLoudong", params, "utf-8") + println("${id} ${houseNum} 程序调用异常") + Thread.sleep(6000) + } + if(result){ + if (result.getInt("code") == 200) { + return result; + } else if (result.getInt("code") !=0) { + println("${id} ${houseNum}") + } + } + + } catch (Exception e) { + println("${id} ${houseNum} ${e.getMessage()}") + e.printStackTrace(); + } + return null + } + + public static JSONObject searchHouseLoudong(String id, String houseNum, String unitNum, String roomNum){ + try{ + Map params = new HashMap(); + params.put("id", id); + params.put("houseNumber", houseNum); + params.put("unitNumber", unitNum); + params.put("roomNumber", roomNum); + params.put("token", TOKEN); + String content = new HttpRequest().get("http://localhost:8080/fangjia-open/property/searchHouseLoudong", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + while (result == null) { + result = HttpRequestUtil.getJsonContent("http://localhost:8080/fangjia-open/property/searchHouseLoudong", params, "utf-8") + println("${id} ${houseNum} 程序调用异常") + Thread.sleep(6000) + } + if(result){ + if (result.getInt("code") == 200) { + return result; + } else if (result.getInt("code") !=0) { + println("${id} ${houseNum}") + } + } + + } catch (Exception e) { + println("${id} ${houseNum} ${e.getMessage()}") + e.printStackTrace(); + } + return null + } + + public static JSONObject standard2Action(String city, String address) { + try { + Map params = new HashMap(); + params.put("city", city); + params.put("address", address); + params.put("token", TOKEN); + params.put("__refresh_cache","1") + + String content = new HttpRequest().get("https://open.fangjia.com/v2/address/standard", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + while (result == null) { + result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/v2/address/standard", params, "utf-8") + println("${city} ${address} 程序调用异常") + Thread.sleep(6000) + } + if (result.getInt("code") == 0) { + return result.getJSONObject("result"); + } else if (result.getInt("code") !=0) { + println("${city} ${address}") + } + } catch (Exception e) { + println("${city} ${address} ${e.getMessage()}") + e.printStackTrace(); + } + return null + } + + public static JSONObject standardize(String city, String address) { try { Map params = new HashMap(); @@ -593,6 +680,90 @@ public class OpenApiClient { } + static JSONObject facilitys(String city, String region, String name) { + Map params = new HashMap(); + params.put("token", TOKEN); + params.put("city", city) + params.put("region", region) + params.put("name", name) + try { + String content = new HttpRequest().get("http://localhost:8080/fangjia-open/district/facilitys", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + + return result + } catch (Exception e) { + + } + return null + } + + static JSONObject searchPaimais(String city, String region, String name, limit) { + Map params = new HashMap(); + params.put("token", TOKEN); + params.put("city", city) + params.put("district", region) + params.put("name", name) + params.put("limit", "10") + try { + String content = new HttpRequest().get("http://localhost:8080/fangjia-open/auction/searchPaimais", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + return result + } catch (Exception e) { + + } + return null + } + + + static JSONObject searchDealPrice(String id, String startDate, limit) { + Map params = new HashMap(); + params.put("token", TOKEN); + params.put("id", id) + params.put("startDate", startDate) + params.put("limit", limit.toString()) + try { + String content = new HttpRequest().get("http://localhost:8080/fangjia-open/dealPrice/searchList", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + return result + } catch (Exception e) { + + } + return null + } + + static JSONObject houseList(String id, String startDate, limit) { + Map params = new HashMap(); + params.put("token", TOKEN); + params.put("id", id) + params.put("startTime", startDate) + params.put("pageSize", limit.toString()) + try { + String content = new HttpRequest().get("http://localhost:8080/fangjia-open/house/list", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + return result + } catch (Exception e) { + + } + return null + } + + + static JSONObject evaluaionList(String id, String startDate, limit) { + Map params = new HashMap(); + params.put("token", TOKEN); + params.put("id", id) + params.put("startDate", startDate) + params.put("limit", limit.toString()) + try { + String content = new HttpRequest().get("http://localhost:8080/fangjia-open/evaluation/bsList", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + return result + } catch (Exception e) { + + } + return null + } + static JSONObject districtDetail(String city, String region, String name) { Map params = new HashMap(); params.put("token", TOKEN); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy index 3810aa9f3..b7fe9aa99 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20220301", "yyyyMMdd") -def endDate = DateUtil.parseDate("20220401", "yyyyMMdd") +def startDate = DateUtil.parseDate("20220401", "yyyyMMdd") +def endDate = DateUtil.parseDate("20220501", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/BsEvaluationService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/BsEvaluationService.java new file mode 100644 index 000000000..733fcb368 --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/BsEvaluationService.java @@ -0,0 +1,11 @@ +package com.fangjia.service.datastore; + +import com.fangjia.dal.mongo.model.BsEvaluation; + +import java.util.Date; +import java.util.List; + +public interface BsEvaluationService { + + List getNearlySameEvaluation(String id, Integer minPrice, Integer maxPrice, Date startDate, Date endDate, Integer limit); +} diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/BsEvaluationServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/BsEvaluationServiceImpl.java new file mode 100644 index 000000000..4d96aaadd --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/BsEvaluationServiceImpl.java @@ -0,0 +1,44 @@ +package com.fangjia.service.datastore; + +import com.fangjia.dal.QueryParam; +import com.fangjia.dal.SortParam; +import com.fangjia.dal.mongo.dao.BsEvaluationDao; +import com.fangjia.dal.mongo.dao.EvaluationDao; +import com.fangjia.dal.mongo.model.BsEvaluation; +import com.fangjia.dal.mongo.model.bkoff.DealPrice; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +public class BsEvaluationServiceImpl implements BsEvaluationService { + + private BsEvaluationDao bsEvaluationDao; + + public void setBsEvaluationDao(BsEvaluationDao bsEvaluationDao) { + this.bsEvaluationDao = bsEvaluationDao; + } + + @Override + public List getNearlySameEvaluation(String id, Integer minPrice, Integer maxPrice, Date startDate, Date endDate, Integer limit) { + List params = new ArrayList(); + params.add(new QueryParam("commid", id)); + if(startDate!=null){ + params.add(new QueryParam("cDate", QueryParam.GREATER_THAN_EQUAL_OPERATOR, startDate)); + } + if(endDate!=null){ + params.add(new QueryParam("cDate", QueryParam.LESS_THAN_EQUAL_OPERATOR, startDate)); + } + + if (minPrice > 0) { + params.add(new QueryParam("unitPrice", QueryParam.GREATER_THAN_OPERATOR, minPrice)); + } + if (maxPrice > 0) { + params.add(new QueryParam("unitPrice", QueryParam.LESS_THAN_OPERATOR, maxPrice)); + } + List sortParams = new ArrayList(); + sortParams.add(new SortParam("cDate", false)); + List bsEvaluations = bsEvaluationDao.findAllByParams(params, sortParams, limit); + return bsEvaluations; + } +} diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/HouseLoudongService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/HouseLoudongService.java index 3f6ad7f39..236cf00c0 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/HouseLoudongService.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/HouseLoudongService.java @@ -94,4 +94,6 @@ public interface HouseLoudongService { public List findAll(List queryParams, Pager pager); List findAllSources(); + + public List findAll(String id, String houseNum, String unitNum, String roomNum); } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/HouseLoudongServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/HouseLoudongServiceImpl.java index 70286758b..9fe6af3ef 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/HouseLoudongServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/HouseLoudongServiceImpl.java @@ -297,6 +297,22 @@ public class HouseLoudongServiceImpl implements HouseLoudongService { return froms; } + @Override + public List findAll(String id, String houseNum, String unitNum, String roomNum) { + List params = new ArrayList(); + params.add(new QueryParam("districtId", id)); + if (StringUtils.isNotBlank(houseNum)) { + params.add(new QueryParam("houseNum", houseNum)); + } + if (StringUtils.isNotBlank(unitNum)) { + params.add(new QueryParam("houseNumUnit", unitNum)); + } + if (StringUtils.isNotBlank(roomNum)) { + params.add(new QueryParam("roomNum", roomNum)); + } + return houseLoudongBkoffDao.findAllByParams(params); + } + @Override public List queryHouseLoudongList(String city, String districtName, String roomNum, Pager pager) { List params = new ArrayList(); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/LoudongService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/LoudongService.java index a38fdab95..d6dc0d63b 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/LoudongService.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/LoudongService.java @@ -23,6 +23,8 @@ public interface LoudongService { List findAll(String city, String region, String name, String houseNumber, String unitNumber, int limit); + List findAll(String id, String houseNumber, String unitNumber); + Map findAll(List query, String city); Loudong findById(String id); @@ -41,4 +43,6 @@ public interface LoudongService { void save(Map data); List findAllSources(); + + Loudong findForEvaluate(String idInSeaweed, String houseNumber, String unitNumber); } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/LoudongServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/LoudongServiceImpl.java index 57e9c6bad..d90620ac3 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/LoudongServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/LoudongServiceImpl.java @@ -148,6 +148,23 @@ public class LoudongServiceImpl implements LoudongService { return loudongDao.findAllByParams(params,null, 0 , limit, ADDRESS_VERIFY_FIELDS); } + @Override + public List findAll(String id, String houseNumber, String unitNumber) { + List params = new ArrayList(); + if(StringUtils.isNotBlank(id)) { + params.add(new QueryParam("idInSeaweed", id)); + } + + if(StringUtils.isNotBlank(houseNumber)) { + params.add(new QueryParam("houseNumber", houseNumber)); + } + if(StringUtils.isNotBlank(unitNumber)) { + params.add(new QueryParam("unitNumber", unitNumber)); + } + + return loudongDao.findAllByParams(params); + } + @Override public Loudong findById(String id) { return loudongDao.findById(id); @@ -318,4 +335,23 @@ public class LoudongServiceImpl implements LoudongService { } return froms; } + + @Override + public Loudong findForEvaluate(String idInSeaweed, String houseNumber, String unitNumber){ + if(StringUtils.isBlank(idInSeaweed) || StringUtils.isBlank(houseNumber)){ + return null; + } + List params = Lists.newArrayList(); + params.add(new QueryParam("idInSeaweed", idInSeaweed)); + params.add(new QueryParam("houseNumber", houseNumber)); + if(StringUtils.isNotBlank(unitNumber)) { + params.add(new QueryParam("unitNumber", unitNumber)); + } + params.add(new QueryParam("floors", QueryParam.GREATER_THAN_OPERATOR, 0)); + params.add(new QueryParam("floors", QueryParam.LESS_THAN_OPERATOR, 60)); + params.add(new QueryParam("status", QueryParam.NOT_EQUAL_OPERATOR, 0)); + + List sortParams = ImmutableList.of(new SortParam("floors", false)); + return loudongDao.findFirstByParams(params, sortParams); + } } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/DealPriceService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/DealPriceService.java index 9061a3288..fbd4dd7ea 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/DealPriceService.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/DealPriceService.java @@ -82,6 +82,8 @@ public interface DealPriceService { public List getNearlySameDealPrice(DistrictKey key, int minPrice, int maxPrice, int radius, int limit); + public List getNearlySameDealPrice(String districtId, Integer minPrice, Integer maxPrice,Date startDate, Date endDate, int limit); + public List getDisSameDealPrice(DistrictKey key, int minPrice, int maxPrice, int limit); /** diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/DealPriceServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/DealPriceServiceImpl.java index be0f7261f..dd205bd77 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/DealPriceServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/DealPriceServiceImpl.java @@ -354,6 +354,29 @@ public class DealPriceServiceImpl implements DealPriceService { return houses; } + @Override + public List getNearlySameDealPrice(String districtId, Integer minPrice, Integer maxPrice, Date startDate, Date endDate, int limit) { + List params = new ArrayList(); + params.add(new QueryParam("matchId", districtId)); + if(startDate!=null){ + params.add(new QueryParam("tradeDate", QueryParam.GREATER_THAN_EQUAL_OPERATOR, startDate)); + } + if(endDate!=null){ + params.add(new QueryParam("tradeDate", QueryParam.LESS_THAN_EQUAL_OPERATOR, startDate)); + } + + if (minPrice > 0) { + params.add(new QueryParam("avgPrice", QueryParam.GREATER_THAN_OPERATOR, minPrice)); + } + if (maxPrice > 0) { + params.add(new QueryParam("avgPrice", QueryParam.LESS_THAN_OPERATOR, maxPrice)); + } + List sortParams = new ArrayList(); + sortParams.add(new SortParam("tradeDate", false)); + List dealPrices = dealPriceDao.findAllByParams(params, sortParams, limit); + return dealPrices; + } + @Override public List getDisSameDealPrice(DistrictKey key, int minPrice, int maxPrice, int limit) { List params = new ArrayList();