Merge branch 'master' of http://gitlab.fangjia.com/dev0/main
This commit is contained in:
@@ -0,0 +1,7 @@
|
||||
package com.fangjia.dal.mongo.dao;
|
||||
|
||||
import com.fangjia.dal.MongoDao;
|
||||
import com.fangjia.dal.mongo.model.BsEvaluation;
|
||||
|
||||
public interface BsEvaluationDao extends MongoDao<BsEvaluation> {
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
package com.fangjia.dal.mongo.dao;
|
||||
|
||||
import com.fangjia.dal.MongoDaoImpl;
|
||||
import com.fangjia.dal.mongo.model.BsEvaluation;
|
||||
|
||||
public class BsEvaluationDaoImpl extends MongoDaoImpl<BsEvaluation> implements
|
||||
BsEvaluationDao {
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
package com.fangjia.dal.mongo.model;
|
||||
|
||||
import com.fangjia.dal.MongoEntity;
|
||||
|
||||
import javax.persistence.*;
|
||||
import java.util.Date;
|
||||
|
||||
@Entity
|
||||
@Table(name = "business.evaluation")
|
||||
public class BsEvaluation implements MongoEntity {
|
||||
private String id;
|
||||
private String fullAddress;
|
||||
private String commid;
|
||||
private String cityName;
|
||||
private String distName;
|
||||
private String commName;
|
||||
private String address;
|
||||
private Double buildArea;
|
||||
private Integer floor;
|
||||
private Integer floorcnt;
|
||||
private Integer roomcnt;
|
||||
private Integer livingroomcnt;
|
||||
private String houseaspect;//ÄÏ
|
||||
private String fitment;
|
||||
private Date cDate;
|
||||
private Integer unitPrice;
|
||||
private Integer totalPrice;
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.AUTO)
|
||||
@Column(name = "_id")
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setId(String id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
@Column(name = "full_address")
|
||||
public String getFullAddress() {
|
||||
return fullAddress;
|
||||
}
|
||||
|
||||
public void setFullAddress(String fullAddress) {
|
||||
this.fullAddress = fullAddress;
|
||||
}
|
||||
|
||||
@Column(name = "commid")
|
||||
public String getCommid() {
|
||||
return commid;
|
||||
}
|
||||
|
||||
public void setCommid(String commid) {
|
||||
this.commid = commid;
|
||||
}
|
||||
|
||||
@Column(name = "cityname")
|
||||
public String getCityName() {
|
||||
return cityName;
|
||||
}
|
||||
|
||||
public void setCityName(String cityName) {
|
||||
this.cityName = cityName;
|
||||
}
|
||||
|
||||
@Column(name = "distname")
|
||||
public String getDistName() {
|
||||
return distName;
|
||||
}
|
||||
|
||||
public void setDistName(String distName) {
|
||||
this.distName = distName;
|
||||
}
|
||||
|
||||
@Column(name = "commname")
|
||||
public String getCommName() {
|
||||
return commName;
|
||||
}
|
||||
|
||||
public void setCommName(String commName) {
|
||||
this.commName = commName;
|
||||
}
|
||||
|
||||
@Column(name = "address")
|
||||
public String getAddress() {
|
||||
return address;
|
||||
}
|
||||
|
||||
public void setAddress(String address) {
|
||||
this.address = address;
|
||||
}
|
||||
|
||||
@Column(name = "buildarea")
|
||||
public Double getBuildArea() {
|
||||
return buildArea;
|
||||
}
|
||||
|
||||
public void setBuildArea(Double buildArea) {
|
||||
this.buildArea = buildArea;
|
||||
}
|
||||
|
||||
@Column(name = "floor")
|
||||
public Integer getFloor() {
|
||||
return floor;
|
||||
}
|
||||
|
||||
public void setFloor(Integer floor) {
|
||||
this.floor = floor;
|
||||
}
|
||||
|
||||
@Column(name = "floorcnt")
|
||||
public Integer getFloorcnt() {
|
||||
return floorcnt;
|
||||
}
|
||||
|
||||
public void setFloorcnt(Integer floorcnt) {
|
||||
this.floorcnt = floorcnt;
|
||||
}
|
||||
|
||||
@Column(name = "roomcnt")
|
||||
public Integer getRoomcnt() {
|
||||
return roomcnt;
|
||||
}
|
||||
|
||||
public void setRoomcnt(Integer roomcnt) {
|
||||
this.roomcnt = roomcnt;
|
||||
}
|
||||
|
||||
@Column(name = "livingroomcnt")
|
||||
public Integer getLivingroomcnt() {
|
||||
return livingroomcnt;
|
||||
}
|
||||
|
||||
public void setLivingroomcnt(Integer livingroomcnt) {
|
||||
this.livingroomcnt = livingroomcnt;
|
||||
}
|
||||
|
||||
@Column(name = "houseaspect")
|
||||
public String getHouseaspect() {
|
||||
return houseaspect;
|
||||
}
|
||||
|
||||
public void setHouseaspect(String houseaspect) {
|
||||
this.houseaspect = houseaspect;
|
||||
}
|
||||
|
||||
@Column(name = "fitment")
|
||||
public String getFitment() {
|
||||
return fitment;
|
||||
}
|
||||
|
||||
public void setFitment(String fitment) {
|
||||
this.fitment = fitment;
|
||||
}
|
||||
|
||||
@Column(name = "c_date")
|
||||
public Date getcDate() {
|
||||
return cDate;
|
||||
}
|
||||
|
||||
public void setcDate(Date cDate) {
|
||||
this.cDate = cDate;
|
||||
}
|
||||
|
||||
@Column(name = "unitprice")
|
||||
public Integer getUnitPrice() {
|
||||
return unitPrice;
|
||||
}
|
||||
|
||||
public void setUnitPrice(Integer unitPrice) {
|
||||
this.unitPrice = unitPrice;
|
||||
}
|
||||
|
||||
@Column(name = "totalprice")
|
||||
public Integer getTotalPrice() {
|
||||
return totalPrice;
|
||||
}
|
||||
|
||||
public void setTotalPrice(Integer totalPrice) {
|
||||
this.totalPrice = totalPrice;
|
||||
}
|
||||
}
|
||||
@@ -63,6 +63,7 @@ public class DistrictFacilitys implements MongoEntity {
|
||||
private String imgUrl3D;
|
||||
|
||||
private List<String> school;
|
||||
private List<String> keySchool;
|
||||
|
||||
@Id
|
||||
@GeneratedValue
|
||||
@@ -380,4 +381,12 @@ public class DistrictFacilitys implements MongoEntity {
|
||||
this.school = school;
|
||||
}
|
||||
|
||||
@Column(name = "key_school")
|
||||
public List<String> getKeySchool() {
|
||||
return keySchool;
|
||||
}
|
||||
|
||||
public void setKeySchool(List<String> keySchool) {
|
||||
this.keySchool = keySchool;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,6 +48,11 @@ public class DealPrice implements MongoEntity {
|
||||
private Integer floor;//当前楼层
|
||||
private Long score;
|
||||
|
||||
private String matchId;
|
||||
private String matchCity;
|
||||
private String matchRegion;
|
||||
private String matchName;
|
||||
|
||||
@Id
|
||||
@GeneratedValue
|
||||
@Column(name = "_id", unique = true, nullable = false)
|
||||
@@ -296,4 +301,40 @@ public class DealPrice implements MongoEntity {
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@Column(name = "match_id")
|
||||
public String getMatchId() {
|
||||
return matchId;
|
||||
}
|
||||
|
||||
public void setMatchId(String matchId) {
|
||||
this.matchId = matchId;
|
||||
}
|
||||
|
||||
@Column(name = "match_city")
|
||||
public String getMatchCity() {
|
||||
return matchCity;
|
||||
}
|
||||
|
||||
public void setMatchCity(String matchCity) {
|
||||
this.matchCity = matchCity;
|
||||
}
|
||||
|
||||
@Column(name = "match_region")
|
||||
public String getMatchRegion() {
|
||||
return matchRegion;
|
||||
}
|
||||
|
||||
public void setMatchRegion(String matchRegion) {
|
||||
this.matchRegion = matchRegion;
|
||||
}
|
||||
|
||||
@Column(name = "match_name")
|
||||
public String getMatchName() {
|
||||
return matchName;
|
||||
}
|
||||
|
||||
public void setMatchName(String matchName) {
|
||||
this.matchName = matchName;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -448,6 +448,7 @@ public class HouseLoudong extends com.fangjia.dal.mongo.model.HouseLoudong{
|
||||
this.openTotalPrice = openTotalPrice;
|
||||
}
|
||||
|
||||
@Column(name = "district_id")
|
||||
public String getDistrictId() {
|
||||
return districtId;
|
||||
}
|
||||
|
||||
@@ -101,7 +101,11 @@ public class Loudong implements MongoEntity {
|
||||
* seaweed中的小区名
|
||||
*/
|
||||
private String nameInSeaweed;
|
||||
|
||||
/**
|
||||
* seaweed中的小区id
|
||||
*/
|
||||
private String idInSeaweed;
|
||||
|
||||
@Column(name = "remark")
|
||||
public String getRemark() {
|
||||
return remark;
|
||||
@@ -402,4 +406,13 @@ public class Loudong implements MongoEntity {
|
||||
public void setNameInSeaweed(String nameInSeaweed) {
|
||||
this.nameInSeaweed = nameInSeaweed;
|
||||
}
|
||||
|
||||
@Column(name = "id_in_seaweed")
|
||||
public String getIdInSeaweed() {
|
||||
return idInSeaweed;
|
||||
}
|
||||
|
||||
public void setIdInSeaweed(String idInSeaweed) {
|
||||
this.idInSeaweed = idInSeaweed;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,6 +67,13 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
|
||||
must.add(addTerms("district_name", alias));
|
||||
must.add(addMatch("cat", "sell"));
|
||||
must.add(addRange("avg_price", minPrice, maxPrice));
|
||||
|
||||
JSONObject term = new JSONObject();
|
||||
JSONObject match = new JSONObject();
|
||||
match.put("p_date", 20210501);
|
||||
term.put("match", match);
|
||||
must.add(term);
|
||||
|
||||
bool.put("must", must);
|
||||
query.put("bool", bool);
|
||||
reqBody.put("query", query);
|
||||
|
||||
@@ -58,6 +58,8 @@ public class Seawater implements SeawaterEntity {
|
||||
private String url;
|
||||
private String root;
|
||||
private Integer avgPrice;
|
||||
private String purpose;
|
||||
private String estateType;
|
||||
|
||||
@Id
|
||||
@Column(name = "_id")
|
||||
@@ -285,4 +287,22 @@ public class Seawater implements SeawaterEntity {
|
||||
public void setAvgPrice(Integer avgPrice) {
|
||||
this.avgPrice = avgPrice;
|
||||
}
|
||||
|
||||
@Column(name = "purpose")
|
||||
public String getPurpose() {
|
||||
return purpose;
|
||||
}
|
||||
|
||||
public void setPurpose(String purpose) {
|
||||
this.purpose = purpose;
|
||||
}
|
||||
|
||||
@Column(name = "estate_type")
|
||||
public String getEstateType() {
|
||||
return estateType;
|
||||
}
|
||||
|
||||
public void setEstateType(String estateType) {
|
||||
this.estateType = estateType;
|
||||
}
|
||||
}
|
||||
|
||||
+23
-2
@@ -23,13 +23,13 @@ import java.util.regex.Pattern;
|
||||
/**
|
||||
* 法拍数据提供接口
|
||||
*/
|
||||
@Authority(role = Role.USER)
|
||||
//@Authority(role = Role.USER)
|
||||
public class AuctionController {
|
||||
|
||||
|
||||
private PaiMaiService paiMaiService;
|
||||
private DistrictNewPriceService districtNewPriceService;
|
||||
private static final String[] SEARCH_FIELDS = new String[]{"marketAvgPrice", "startDate", "curPrice", "imgUrls", "id", "matchCity", "matchRegion", "matchName", "address", "marketTotalPrice", "startPrice", "curPrice", "predictPrice", "depositPrice", "area", "status", "url", "matchEstateType"};
|
||||
private static final String[] SEARCH_FIELDS = new String[]{"marketAvgPrice", "startDate", "curPrice", "imgUrls", "id", "matchCity", "matchRegion", "matchName", "address", "marketTotalPrice", "startPrice", "curPrice", "predictPrice", "depositPrice", "area", "status", "url", "matchEstateType","type","count"};
|
||||
|
||||
public Object housesAction(AuctionModel model){
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
@@ -187,6 +187,27 @@ public class AuctionController {
|
||||
return mav;
|
||||
}
|
||||
|
||||
|
||||
public Object searchPaimaisAction(AuctionModel model) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
List<QueryParam> params = new ArrayList<QueryParam>();
|
||||
if(StringUtils.isNotBlank(model.getCity())){
|
||||
params.add(new QueryParam("matchCity", model.getCity()));
|
||||
}
|
||||
if(StringUtils.isNotBlank(model.getDistrict())){
|
||||
params.add(new QueryParam("matchRegion", model.getDistrict()));
|
||||
}
|
||||
|
||||
if(StringUtils.isNotBlank(model.getName())){
|
||||
params.add(new QueryParam("matchName", model.getName()));
|
||||
}
|
||||
List<SortParam> sortParams = new ArrayList<SortParam>();
|
||||
sortParams.add(new SortParam("startDate", false));
|
||||
List<PaiMai> paimais = paiMaiService.findAll(params, sortParams, model.getLimit(), SEARCH_FIELDS);
|
||||
mav.setModel(BasicModel.buildOKModel().setResult(paimais));
|
||||
return mav;
|
||||
}
|
||||
|
||||
public void setPaiMaiService(PaiMaiService paiMaiService) {
|
||||
this.paiMaiService = paiMaiService;
|
||||
}
|
||||
|
||||
+38
@@ -15,8 +15,10 @@ import java.util.Map.Entry;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import com.fangjia.dal.common.DistrictKey;
|
||||
import com.fangjia.dal.mongo.model.Metro;
|
||||
import com.fangjia.open.model.BasicModel;
|
||||
import com.fangjia.service.bo.DistrictStatBo;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
import com.fangjia.core.util.DateUtil;
|
||||
@@ -211,6 +213,42 @@ public class DealPriceController {
|
||||
mv.setModel(resMap);
|
||||
return mv;
|
||||
}
|
||||
|
||||
public Object searchListAction(final DealPriceModel model) {
|
||||
RestfulModelAndView mv = new RestfulModelAndView();
|
||||
mv.setViewType("rest");
|
||||
|
||||
if (StringUtils.isBlank(model.getId())) {
|
||||
mv.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "楼盘编号不能为空"));
|
||||
return mv;
|
||||
}
|
||||
District district = districtService.findDistrictById(model.getId());
|
||||
DistrictKey districtKey = new DistrictKey(district.getCity(), district.getRegion(), district.getName());
|
||||
|
||||
if(district == null){
|
||||
mv.setModel(BasicModel.buildBadRequestModel("该小区不存在"));
|
||||
return mv;
|
||||
}
|
||||
DistrictStatBo bo = districtStatService.findCurrentDistrictStat(districtKey);
|
||||
Integer minPrice = null;
|
||||
Integer maxPrice = null;
|
||||
if(bo.getDistrictStat()!=null){
|
||||
minPrice = (int)(bo.getDistrictStat().getAvgPrice() * 0.8);
|
||||
maxPrice = (int)(bo.getDistrictStat().getAvgPrice() * 1.2);
|
||||
}
|
||||
Date startDate = null;
|
||||
Date endDate = null;
|
||||
if(StringUtils.isNotBlank(model.getStartDate())){
|
||||
startDate = DateUtil.parseDate(model.getStartDate(),"yyyyMMdd");
|
||||
}
|
||||
if(StringUtils.isNotBlank(model.getEndDate())){
|
||||
endDate = DateUtil.parseDate(model.getEndDate(),"yyyyMMdd");
|
||||
}
|
||||
|
||||
List<DealPrice> dealPrices = dealPriceService.getNearlySameDealPrice(model.getId(), minPrice, maxPrice, startDate, endDate, model.getLimit());
|
||||
mv.setModel(BasicModel.buildOKModel().setResult(dealPrices));
|
||||
return mv;
|
||||
}
|
||||
|
||||
private List<DealPrice> findDealPricesByName(String city,String region ,String name){
|
||||
Calendar calendar = Calendar.getInstance();
|
||||
|
||||
+37
-16
@@ -7,11 +7,11 @@ import javax.servlet.http.HttpServletRequest;
|
||||
|
||||
import com.fangjia.core.RegionData;
|
||||
import com.fangjia.core.util.DateUtil;
|
||||
import com.fangjia.dal.mongo.model.RegionStat;
|
||||
import com.fangjia.dal.mongo.model.RegionStatRent;
|
||||
import com.fangjia.dal.mongo.model.*;
|
||||
import com.fangjia.open.model.AvgPriceQueryModel;
|
||||
import com.fangjia.open.model.BasicModel;
|
||||
import com.fangjia.service.datastore.RegionStatService;
|
||||
import com.fangjia.service.district.DistrictFacilitysService;
|
||||
import com.fangjia.service.district.DistrictStatRentService;
|
||||
import com.fangjia.service.district.RegionStatRentService;
|
||||
import com.fangjia.service.mongo.fangjia.AverageAnalyService;
|
||||
@@ -38,8 +38,6 @@ import com.fangjia.dal.common.DistrictKey;
|
||||
import com.fangjia.dal.common.FangjiaServerAreaContextHolder;
|
||||
import com.fangjia.dal.common.ServerAreaData;
|
||||
import com.fangjia.dal.mongo.fangjia.model.ServerArea;
|
||||
import com.fangjia.dal.mongo.model.DistrictHouse;
|
||||
import com.fangjia.dal.mongo.model.DistrictStat;
|
||||
import com.fangjia.dal.mongo.model.bkoff.ExtractDistrict;
|
||||
import com.fangjia.dal.seaweed.model.District;
|
||||
import com.fangjia.dal.sql.model.ApiUser;
|
||||
@@ -93,6 +91,7 @@ public class DistrictController {
|
||||
private RegionBlockService regionBlockService;
|
||||
private RegionStatService regionStatService;
|
||||
private RegionStatRentService regionStatRentService;
|
||||
private DistrictFacilitysService districtFacilitysService;
|
||||
|
||||
|
||||
public static final String[] COLUMNS = { "id", "name", "estateType", "buildType", "licence", "buildArea", "allArea",
|
||||
@@ -444,19 +443,19 @@ public class DistrictController {
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public Object showAction(DistrictSearchModel model, @Param(name = "city") String city, @Param(name = "region") String region,
|
||||
@Param(name = "name") String name, HttpServletRequest request) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
if (StringUtils.isBlank(name) || StringUtils.isBlank(city) || StringUtils.isBlank(region)) {
|
||||
mav.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "小区所在城市,小区名,所在区域必必须输入"));
|
||||
return mav;
|
||||
} else {
|
||||
DistrictBo districtBo = districtService.findByFullMode(new DistrictKey(city, region, name));
|
||||
if (null == districtBo) {
|
||||
mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "小区数据不存在,请检查查询条件"));
|
||||
public Object showAction(DistrictSearchModel model, @Param(name = "city") String city, @Param(name = "region") String region,
|
||||
@Param(name = "name") String name, HttpServletRequest request) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
if (StringUtils.isBlank(name) || StringUtils.isBlank(city) || StringUtils.isBlank(region)) {
|
||||
mav.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "小区所在城市,小区名,所在区域必必须输入"));
|
||||
return mav;
|
||||
}
|
||||
} else {
|
||||
DistrictBo districtBo = districtService.findByFullMode(new DistrictKey(city, region, name));
|
||||
if (null == districtBo) {
|
||||
mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "小区数据不存在,请检查查询条件"));
|
||||
return mav;
|
||||
}
|
||||
districtBo.setStat(this.districtStatService.findDistrictStat(districtBo.getDistrictKey()));
|
||||
// 去除小区描述中的html标签
|
||||
districtBo.setDescription(DataUtil.removeSpecialString(districtBo.getDescription()));
|
||||
@@ -1146,6 +1145,25 @@ public class DistrictController {
|
||||
ret.put("RESULT", 200);
|
||||
return mav;
|
||||
}
|
||||
|
||||
public Object facilitysAction(DistrictSearchModel model, @Param(name = "city") String city, @Param(name = "region") String region,
|
||||
@Param(name = "name") String name, HttpServletRequest request) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
if (StringUtils.isBlank(name) || StringUtils.isBlank(city) || StringUtils.isBlank(region)) {
|
||||
mav.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "小区所在城市,小区名,所在区域必必须输入"));
|
||||
return mav;
|
||||
} else {
|
||||
District districtBo = districtService.findDistrict(new DistrictKey(city, region, name), "district");
|
||||
if (null == districtBo) {
|
||||
mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "小区数据不存在"));
|
||||
return mav;
|
||||
}
|
||||
DistrictFacilitys facilitys = districtFacilitysService.findByDistrict(districtBo);
|
||||
mav.setModel(BasicModel.buildOKModel().setResult(facilitys));
|
||||
}
|
||||
return mav;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -1230,4 +1248,7 @@ public class DistrictController {
|
||||
this.regionStatRentService = regionStatRentService;
|
||||
}
|
||||
|
||||
public void setDistrictFacilitysService(DistrictFacilitysService districtFacilitysService) {
|
||||
this.districtFacilitysService = districtFacilitysService;
|
||||
}
|
||||
}
|
||||
|
||||
+57
-5
@@ -10,9 +10,14 @@ import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.fangjia.dal.mongo.model.BsEvaluation;
|
||||
import com.fangjia.dal.mongo.model.bkoff.DealPrice;
|
||||
import com.fangjia.service.bo.AddrEvaResult;
|
||||
import com.fangjia.service.bo.DistrictStatBo;
|
||||
import com.fangjia.service.bo.EvaluationBo;
|
||||
import com.fangjia.service.bo.HouseScorer;
|
||||
import com.fangjia.service.datastore.*;
|
||||
import com.fangjia.service.district.DistrictStatService;
|
||||
import org.apache.commons.collections.MapUtils;
|
||||
import org.apache.commons.lang.ArrayUtils;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
@@ -54,11 +59,6 @@ import com.fangjia.open.model.RestfulModelAndView;
|
||||
import com.fangjia.service.api.AutoCompleteService;
|
||||
import com.fangjia.service.api.KnowbaseService;
|
||||
import com.fangjia.service.common.SellSimilar;
|
||||
import com.fangjia.service.datastore.BusinessEstateService;
|
||||
import com.fangjia.service.datastore.DistrictService;
|
||||
import com.fangjia.service.datastore.EvaluationReportService;
|
||||
import com.fangjia.service.datastore.EvaluationService;
|
||||
import com.fangjia.service.datastore.HouseService;
|
||||
import com.fangjia.service.sql.ApiUserService;
|
||||
import com.fangjia.service.sql.UserService;
|
||||
import com.fangjia.service.web.AddressService;
|
||||
@@ -94,6 +94,8 @@ public class EvaluationController {
|
||||
private ApiUserService apiUserService;
|
||||
private UserService userService;
|
||||
private AddressService addressService;
|
||||
private DistrictStatService districtStatService;
|
||||
private BsEvaluationService bsEvaluationService;
|
||||
|
||||
private final static String AK = OpenConfig.getConfig().getBaiduAk();
|
||||
public Object lngLatEvaluateAction(EvaluationModel model) {
|
||||
@@ -1005,6 +1007,44 @@ public class EvaluationController {
|
||||
return mav;
|
||||
}
|
||||
|
||||
public Object bsListAction(EvaluationModel model) {
|
||||
RestfulModelAndView mv = new RestfulModelAndView();
|
||||
mv.setViewType("rest");
|
||||
|
||||
if (StringUtils.isBlank(model.getId())) {
|
||||
mv.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "楼盘编号不能为空"));
|
||||
return mv;
|
||||
}
|
||||
District district = districtService.findDistrictById(model.getId());
|
||||
DistrictKey districtKey = new DistrictKey(district.getCity(), district.getRegion(), district.getName());
|
||||
|
||||
if(district == null){
|
||||
mv.setModel(BasicModel.buildBadRequestModel("该小区不存在"));
|
||||
return mv;
|
||||
}
|
||||
DistrictStatBo bo = districtStatService.findCurrentDistrictStat(districtKey);
|
||||
Integer minPrice = null;
|
||||
Integer maxPrice = null;
|
||||
if(bo.getDistrictStat()!=null){
|
||||
minPrice = (int)(bo.getDistrictStat().getAvgPrice() * 0.8);
|
||||
maxPrice = (int)(bo.getDistrictStat().getAvgPrice() * 1.2);
|
||||
}
|
||||
Date startDate = null;
|
||||
Date endDate = null;
|
||||
if(StringUtils.isNotBlank(model.getStartDate())){
|
||||
startDate = DateUtil.parseDate(model.getStartDate(),"yyyyMMdd");
|
||||
}
|
||||
if(StringUtils.isNotBlank(model.getEndDate())){
|
||||
endDate = DateUtil.parseDate(model.getEndDate(),"yyyyMMdd");
|
||||
}
|
||||
|
||||
List<BsEvaluation> dealPrices = bsEvaluationService.getNearlySameEvaluation(model.getId(), minPrice, maxPrice, startDate, endDate, model.getLimit());
|
||||
mv.setModel(BasicModel.buildOKModel().setResult(dealPrices));
|
||||
return mv;
|
||||
}
|
||||
|
||||
|
||||
|
||||
public void setEvaluationService(EvaluationService evaluationService) {
|
||||
this.evaluationService = evaluationService;
|
||||
}
|
||||
@@ -1017,6 +1057,18 @@ public class EvaluationController {
|
||||
this.knowbaseService = knowbaseService;
|
||||
}
|
||||
|
||||
|
||||
|
||||
public void setDistrictStatService(DistrictStatService districtStatService) {
|
||||
this.districtStatService = districtStatService;
|
||||
}
|
||||
|
||||
|
||||
|
||||
public void setBsEvaluationService(BsEvaluationService bsEvaluationService) {
|
||||
this.bsEvaluationService = bsEvaluationService;
|
||||
}
|
||||
|
||||
public void setAutoCompleteService(AutoCompleteService autoCompleteService) {
|
||||
this.autoCompleteService = autoCompleteService;
|
||||
}
|
||||
|
||||
@@ -11,6 +11,8 @@ import com.fangjia.dal.mongo.model.DistrictFacilitys;
|
||||
import com.fangjia.dal.mongo.model.FangjiaOfficialAreaMap;
|
||||
import com.fangjia.dal.mongo.model.Metro;
|
||||
import com.fangjia.dal.mongo.model.StatsIndex;
|
||||
import com.fangjia.dal.seawater.dao.SeawaterDao;
|
||||
import com.fangjia.dal.seawater.model.Seawater;
|
||||
import com.fangjia.dal.seaweed.model.District;
|
||||
import com.fangjia.dal.sql.model.ApiUser;
|
||||
import com.fangjia.datastore.driver.FacetResult;
|
||||
@@ -29,6 +31,7 @@ import com.fangjia.open.ws.query.CollateralSampleQuery;
|
||||
import com.fangjia.open.ws.query.ShopCollateralSampleQuery;
|
||||
import com.fangjia.service.address.AddressParser;
|
||||
import com.fangjia.service.bo.Collateral;
|
||||
import com.fangjia.service.bo.DistrictStatBo;
|
||||
import com.fangjia.service.bo.Sample;
|
||||
import com.fangjia.service.bo.ShopSample;
|
||||
import com.fangjia.service.datastore.CzbankSampleService;
|
||||
@@ -39,11 +42,13 @@ import com.fangjia.service.datastore.bean.CollateralSampleQueryBean;
|
||||
import com.fangjia.service.datastore.bean.ShopCollateralSampleQueryBean;
|
||||
import com.fangjia.service.district.DistrictFacilitysService;
|
||||
import com.fangjia.service.district.DistrictStatService;
|
||||
import com.fangjia.service.district.SqmetreRentService;
|
||||
import com.fangjia.service.menu.Item;
|
||||
import com.fangjia.service.menu.MenuCategoryEnum;
|
||||
import com.fangjia.service.menu.Value;
|
||||
import com.fangjia.service.mongo.fangjia.DealPriceService;
|
||||
import com.fangjia.service.mongo.fangjia.MetroService;
|
||||
import com.fangjia.service.mongo.fangjia.SeawaterService;
|
||||
import com.fangjia.service.mongo.fangjia.StatsIndexService;
|
||||
import com.fangjia.service.open.FangjiaOfficialAreaService;
|
||||
import com.fangjia.service.open.HousePojo;
|
||||
@@ -86,6 +91,9 @@ public class HouseController {
|
||||
private CzbankSampleService czbankSampleService;
|
||||
private DistrictService districtService;
|
||||
private FangjiaOfficialAreaService fangjiaOfficialAreaService;
|
||||
|
||||
private SeawaterService seawaterService;
|
||||
private SeawaterDao seawaterDao;
|
||||
private List<String> PROPERTY_TYPE_CODE_LIST = Arrays.asList("01", "02");
|
||||
|
||||
|
||||
@@ -399,6 +407,49 @@ public class HouseController {
|
||||
}
|
||||
}
|
||||
|
||||
public Object listAction(final HouseSearchModel model) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
if (StringUtils.isBlank(model.getId()) && (StringUtils.isBlank(model.getCity()) || StringUtils.isBlank(model.getRegion()) || StringUtils.isBlank(model.getName()))) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("楼盘编号或城市,区域,名称为必填项"));
|
||||
return mav;
|
||||
}
|
||||
District district = null;
|
||||
DistrictKey districtKey = null;
|
||||
if(StringUtils.isNotBlank(model.getName())){
|
||||
districtKey = new DistrictKey(model.getCity(), model.getRegion(), model.getName());
|
||||
district = districtService.findDistrict(districtKey, "district");
|
||||
}else{
|
||||
district = districtService.findDistrictById(model.getId());
|
||||
districtKey = new DistrictKey(district.getCity(), district.getRegion(), district.getName());
|
||||
}
|
||||
|
||||
if(district == null){
|
||||
mav.setModel(BasicModel.buildBadRequestModel("该小区不存在"));
|
||||
return mav;
|
||||
}
|
||||
DistrictStatBo bo = districtStatService.findCurrentDistrictStat(districtKey);
|
||||
Integer minPrice = null;
|
||||
Integer maxPrice = null;
|
||||
if(bo.getDistrictStat()!=null){
|
||||
minPrice = (int)(bo.getDistrictStat().getAvgPrice() * 0.8);
|
||||
maxPrice = (int)(bo.getDistrictStat().getAvgPrice() * 1.2);
|
||||
}
|
||||
List<String> names = new ArrayList<String>();
|
||||
names.add(district.getName());
|
||||
if(district.getAlias() != null){
|
||||
for(String alia : district.getAlias()){
|
||||
if(StringUtils.isNotBlank(alia) && !names.contains(alia)){
|
||||
names.add(alia);
|
||||
}
|
||||
}
|
||||
}
|
||||
List<Seawater> houses = seawaterDao.findSeawatersFromEs("sell", district.getCity(), district.getRegion(), district.getName(), district.getAlias(), null, model.getStartTime(), model.getEndTime(),null,null,minPrice, maxPrice,null,null,"p_date", false,0, model.getPageSize(),"latest_seawater");
|
||||
//List<Seawater> houses1 = seawaterDao.findHouses(district.getCity(), district.getRegion(), names, minPrice, maxPrice, model.getPageSize());
|
||||
mav.setModel(BasicModel.buildOKModel().setResult(houses));
|
||||
return mav;
|
||||
}
|
||||
|
||||
String getPicUrl(String url, List<String> urls) {
|
||||
if (!urls.contains(url)) {
|
||||
urls.add(url);
|
||||
@@ -832,6 +883,13 @@ public class HouseController {
|
||||
return ret;
|
||||
}
|
||||
|
||||
public void setSeawaterService(SeawaterService seawaterService) {
|
||||
this.seawaterService = seawaterService;
|
||||
}
|
||||
|
||||
public void setSeawaterDao(SeawaterDao seawaterDao) {
|
||||
this.seawaterDao = seawaterDao;
|
||||
}
|
||||
|
||||
private boolean checkExists(FacetQueryResult<HousePojo> houseBoList, HousePojo house) {
|
||||
for (HousePojo bo : houseBoList) {
|
||||
|
||||
+40
-2
@@ -665,6 +665,7 @@ public class PropertyController {
|
||||
return mav;
|
||||
}
|
||||
String userId = FangjiaApiUtil.parseUserId(model.getToken());
|
||||
|
||||
District district = null;
|
||||
if (StringUtils.isNotBlank(model.getId())){
|
||||
if(ObjectId.isValid(model.getId())){
|
||||
@@ -728,7 +729,7 @@ public class PropertyController {
|
||||
} else {
|
||||
districtInfoDto.setBuildingType("");
|
||||
}
|
||||
districtInfoDto.setAlias(district.getAlias());
|
||||
districtInfoDto.setAlias(district.getMappingAlias());
|
||||
districtInfoDto.setAddressAlias(district.getAddrAlias());
|
||||
districtInfoDto.setCommunity(district.getStreetCommunity());
|
||||
districtInfoDto.setCommercialArea(district.getBusinessQuarter());
|
||||
@@ -780,6 +781,7 @@ public class PropertyController {
|
||||
districtInfoDto.setDescription(district.getDescription());
|
||||
districtInfoDto.setSalesOfficeAdd(district.getSellAddress());
|
||||
districtInfoDto.setSalesPhoneNumber(district.getMobile());
|
||||
districtInfoDto.setTownship(district.getTownship());
|
||||
if (district.getLoudongCount() != null && district.getLoudongCount() > 0) {
|
||||
districtInfoDto.setBuildingCount(district.getLoudongCount());
|
||||
}
|
||||
@@ -948,6 +950,12 @@ public class PropertyController {
|
||||
return mav;
|
||||
}
|
||||
|
||||
if(model.getMaxFloor() == null && model.getFloor() != null &&StringUtils.isNotBlank(model.getBuildingNumber())){
|
||||
Loudong loudong = loudongService.findForEvaluate(district.getId(), model.getBuildingNumber(), model.getUnitNumber());
|
||||
if(loudong != null && loudong.getFloors() >= model.getFloor()){
|
||||
model.setMaxFloor(loudong.getFloors());
|
||||
}
|
||||
}
|
||||
model.setRegion(district.getRegion());
|
||||
model.setDistrictName(district.getName());
|
||||
if (StringUtils.isBlank(model.getPropertyType())) {
|
||||
@@ -1104,7 +1112,7 @@ public class PropertyController {
|
||||
model.setCalDate(calDate);
|
||||
}
|
||||
/* 先到CPDB库中查找小区 */
|
||||
District district = matchDistrict(model.getCity(), model.getAddress(), model.getDistrict(), model.getName());
|
||||
District district = districtService.findDistrict(new DistrictKey(model.getCity(), model.getDistrict(), model.getName()), "district");//matchDistrict(model.getCity(), model.getAddress(), model.getDistrict(), model.getName());
|
||||
if(district == null){
|
||||
//防止地址模糊,名称详细的情况
|
||||
district = matchDistrict(model.getCity(), model.getName(), model.getDistrict(), model.getAddress());
|
||||
@@ -5400,6 +5408,36 @@ public class PropertyController {
|
||||
}
|
||||
}
|
||||
|
||||
public Object searchHouseLoudongAction(PropertyEvaluationModel model, HttpServletRequest request)throws NumberFormatException, IllegalAccessException {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
|
||||
if (StringUtils.isBlank(model.getId()) || StringUtils.isBlank(model.getHouseNumber()) || StringUtils.isBlank(model.getRoomNumber())){
|
||||
mav.setModel(BasicModel.buildBadRequestModel("参数非法:楼盘编号,楼栋号,室为必填项"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
List<HouseLoudong> houseLoudongs = houseLoudongService.findAll(model.getId(), model.getHouseNumber(), model.getUnitNumber(), model.getRoomNumber());
|
||||
mav.setModel(BasicModel.buildOKModel().setResult(houseLoudongs));
|
||||
return mav;
|
||||
}
|
||||
|
||||
|
||||
public Object searchLoudongAction(PropertyEvaluationModel model, HttpServletRequest request)throws NumberFormatException, IllegalAccessException {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
|
||||
if (StringUtils.isBlank(model.getId()) || StringUtils.isBlank(model.getHouseNumber())){
|
||||
mav.setModel(BasicModel.buildBadRequestModel("参数非法:楼盘编号,楼栋号为必填项"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
|
||||
List<Loudong> loudongs = loudongService.findAll(model.getId(), model.getHouseNumber(), model.getUnitNumber());
|
||||
mav.setModel(BasicModel.buildOKModel().setResult(loudongs));
|
||||
return mav;
|
||||
}
|
||||
|
||||
|
||||
public void setHouseLoudongService(HouseLoudongService houseLoudongService) {
|
||||
this.houseLoudongService = houseLoudongService;
|
||||
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
package com.fangjia.open.controllers;
|
||||
|
||||
import com.fangjia.core.util.AnalyticsUtil;
|
||||
import com.fangjia.open.interceptors.Authority;
|
||||
import com.fangjia.open.interceptors.Role;
|
||||
import com.fangjia.open.model.BasicModel;
|
||||
import com.fangjia.open.model.RestfulModelAndView;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import java.util.Enumeration;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
public class requestController {
|
||||
|
||||
/**
|
||||
* »ñÈ¡ÇëÇóµÄrequestºÍIPÐÅÏ¢
|
||||
* @param request
|
||||
* @return
|
||||
*/
|
||||
public Object requestAction(HttpServletRequest request) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
String ip = AnalyticsUtil.getIpAddress(request);
|
||||
Map<String, String> map = new HashMap<String, String>();
|
||||
Enumeration headerNames = request.getHeaderNames();
|
||||
while (headerNames.hasMoreElements()) {
|
||||
String key = (String) headerNames.nextElement();
|
||||
String value = request.getHeader(key);
|
||||
map.put(key, value);
|
||||
}
|
||||
map.put("ip", ip);
|
||||
mav.setModel(BasicModel.buildOKModel().setResult(map));
|
||||
return mav;
|
||||
}
|
||||
}
|
||||
@@ -99,6 +99,7 @@ public class DistrictInfoDto {
|
||||
private String salesPhoneNumber;
|
||||
private Integer buildingCount;
|
||||
private Integer finishYear;
|
||||
private String township;
|
||||
|
||||
|
||||
public String getDistrict() {
|
||||
@@ -818,4 +819,12 @@ public class DistrictInfoDto {
|
||||
public void setFinishYear(Integer finishYear) {
|
||||
this.finishYear = finishYear;
|
||||
}
|
||||
|
||||
public String getTownship() {
|
||||
return township;
|
||||
}
|
||||
|
||||
public void setTownship(String township) {
|
||||
this.township = township;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ public class AuctionModel extends FangjiaApiModel{
|
||||
private String city;
|
||||
private String district;
|
||||
private String address;
|
||||
private String name;
|
||||
private Date minStartTime;
|
||||
private Date maxStartTime;
|
||||
private Integer minStartPrice;
|
||||
@@ -121,4 +122,12 @@ public class AuctionModel extends FangjiaApiModel{
|
||||
public void setStatus(Integer status) {
|
||||
this.status = status;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
package com.fangjia.open.model;
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
public class DealPriceModel extends FangjiaApiModel{
|
||||
private String id;
|
||||
|
||||
private String city;
|
||||
|
||||
@@ -34,9 +37,13 @@ public class DealPriceModel extends FangjiaApiModel{
|
||||
private Double size;
|
||||
|
||||
private Integer max;
|
||||
private Integer limit = 10;
|
||||
private String decoration;
|
||||
|
||||
private boolean convert;
|
||||
|
||||
private String startDate;
|
||||
private String endDate;
|
||||
|
||||
|
||||
public String getCity() {
|
||||
@@ -198,4 +205,36 @@ public class DealPriceModel extends FangjiaApiModel{
|
||||
public void setConvert(boolean convert) {
|
||||
this.convert = convert;
|
||||
}
|
||||
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public void setId(String id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public String getStartDate() {
|
||||
return startDate;
|
||||
}
|
||||
|
||||
public void setStartDate(String startDate) {
|
||||
this.startDate = startDate;
|
||||
}
|
||||
|
||||
public String getEndDate() {
|
||||
return endDate;
|
||||
}
|
||||
|
||||
public void setEndDate(String endDate) {
|
||||
this.endDate = endDate;
|
||||
}
|
||||
|
||||
public Integer getLimit() {
|
||||
return limit;
|
||||
}
|
||||
|
||||
public void setLimit(Integer limit) {
|
||||
this.limit = limit;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -246,6 +246,42 @@ public class EvaluationModel extends FangjiaApiModel {
|
||||
private Date buildDate;
|
||||
private String sid;//微信小程序中的sid
|
||||
private String code;//微信小程序中的code
|
||||
private String id;
|
||||
private String startDate;
|
||||
private String endDate;
|
||||
private Integer limit = 10;
|
||||
|
||||
public Integer getLimit() {
|
||||
return limit;
|
||||
}
|
||||
|
||||
public void setLimit(Integer limit) {
|
||||
this.limit = limit;
|
||||
}
|
||||
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public void setId(String id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public String getStartDate() {
|
||||
return startDate;
|
||||
}
|
||||
|
||||
public void setStartDate(String startDate) {
|
||||
this.startDate = startDate;
|
||||
}
|
||||
|
||||
public String getEndDate() {
|
||||
return endDate;
|
||||
}
|
||||
|
||||
public void setEndDate(String endDate) {
|
||||
this.endDate = endDate;
|
||||
}
|
||||
|
||||
public String getRegion() {
|
||||
return region;
|
||||
|
||||
@@ -4,6 +4,7 @@ import com.fangjia.service.datastore.bean.HouseQueryBean;
|
||||
import com.fangjia.service.query.QueryBean;
|
||||
|
||||
public class HouseSearchModel extends SearchModel {
|
||||
private String id;
|
||||
private String kind;
|
||||
private String estateType;
|
||||
private String buildingType;
|
||||
@@ -116,7 +117,12 @@ public class HouseSearchModel extends SearchModel {
|
||||
public void setShowDetail(Boolean showDetail) {
|
||||
this.showDetail = showDetail;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public void setId(String id) {
|
||||
this.id = id;
|
||||
}
|
||||
}
|
||||
|
||||
+21
@@ -46,7 +46,12 @@ public class PropertyEvaluationModel extends FangjiaApiModel {
|
||||
* Â¥ºÅ
|
||||
*/
|
||||
private String unitNumber;
|
||||
/**
|
||||
* ÐÂÂ¥ºÅ
|
||||
*/
|
||||
private String houseNumber;
|
||||
|
||||
private String id;
|
||||
/**
|
||||
* ÊÒºÅ
|
||||
*/
|
||||
@@ -905,4 +910,20 @@ public class PropertyEvaluationModel extends FangjiaApiModel {
|
||||
public void setAvgPrice(Integer avgPrice) {
|
||||
this.avgPrice = avgPrice;
|
||||
}
|
||||
|
||||
public String getHouseNumber() {
|
||||
return houseNumber;
|
||||
}
|
||||
|
||||
public void setHouseNumber(String houseNumber) {
|
||||
this.houseNumber = houseNumber;
|
||||
}
|
||||
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public void setId(String id) {
|
||||
this.id = id;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,8 @@ public abstract class SearchModel extends FangjiaApiModel {
|
||||
protected Integer totalPriceEnd;
|
||||
protected Integer avgPriceStart;
|
||||
protected Integer avgPriceEnd;
|
||||
protected Integer startTime;
|
||||
protected Integer endTime;
|
||||
protected Integer areaStart;
|
||||
protected Integer areaEnd;
|
||||
protected Integer room;
|
||||
@@ -358,4 +360,20 @@ public abstract class SearchModel extends FangjiaApiModel {
|
||||
public void setTotalPrice(Integer totalPrice) {
|
||||
this.totalPrice = totalPrice;
|
||||
}
|
||||
|
||||
public Integer getStartTime() {
|
||||
return startTime;
|
||||
}
|
||||
|
||||
public void setStartTime(Integer startTime) {
|
||||
this.startTime = startTime;
|
||||
}
|
||||
|
||||
public Integer getEndTime() {
|
||||
return endTime;
|
||||
}
|
||||
|
||||
public void setEndTime(Integer endTime) {
|
||||
this.endTime = endTime;
|
||||
}
|
||||
}
|
||||
|
||||
+49
-35
@@ -68,20 +68,30 @@ class BigdataBuildingBatch extends BaseBatch{
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("status", 1)
|
||||
// q.put("name","ÁÉÔ´Ò»´å")
|
||||
def dbcursor = LOU_DONG_STAT.find(q)
|
||||
def map = [:]
|
||||
def set = new HashSet()
|
||||
for(def db : dbcursor){
|
||||
|
||||
def list = map.get(db.region+"##"+db.name)
|
||||
if(!list){
|
||||
list = []
|
||||
map.put(db.region+"##"+db.name, list)
|
||||
}
|
||||
list << db
|
||||
if(db.house_num && db.house_num!="--"){
|
||||
def key = db.region+"##"+db.name +"##"+db.house_num
|
||||
if(!set.contains(key)){
|
||||
set << key
|
||||
list << db
|
||||
}
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
def createCityDatas(cityCodeDatas, distCodeMap, distMap, buildingMap){
|
||||
def rets = []
|
||||
def keys = []
|
||||
distMap.each{k,dist->
|
||||
def buildings = buildingMap.get(k)
|
||||
if(buildings){
|
||||
@@ -89,53 +99,57 @@ class BigdataBuildingBatch extends BaseBatch{
|
||||
def ret = []
|
||||
def areaData = distCodeMap.get(obj.region)
|
||||
if(areaData){
|
||||
ret << cityCodeDatas.get(obj.city)
|
||||
ret << obj._id.toString()
|
||||
ret << obj.house_num
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << areaData.district_code
|
||||
ret << dist._id.toString()
|
||||
if(obj.build_year){
|
||||
ret << obj.build_year
|
||||
}else{
|
||||
def key = dist._id.toString() +"##"+obj.house_num
|
||||
if(!keys.contains(key)){
|
||||
keys << key
|
||||
ret << cityCodeDatas.get(obj.city)
|
||||
ret << obj._id.toString()
|
||||
ret << obj.house_num
|
||||
ret << obj.house_num
|
||||
ret << ""
|
||||
}
|
||||
def arrs = CcbCodeUtils.parseBuildType(dist.build_type)
|
||||
if(arrs){
|
||||
ret << arrs[0].trim()
|
||||
}else{
|
||||
ret << ''
|
||||
}
|
||||
ret << areaData.district_code
|
||||
ret << dist._id.toString()
|
||||
if(obj.build_year){
|
||||
ret << obj.build_year
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
def arrs = CcbCodeUtils.parseBuildType(dist.build_type)
|
||||
if(arrs){
|
||||
ret << arrs[0].trim()
|
||||
}else{
|
||||
ret << ''
|
||||
}
|
||||
|
||||
ret << CcbCodeUtils.getBuildStructureCode(dist.build_structure)
|
||||
if(obj.floors){
|
||||
ret << obj.floors
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << CcbCodeUtils.getBuildStructureCode(dist.build_structure)
|
||||
if(obj.floors){
|
||||
ret << obj.floors
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
// if(obj.elevators && obj.elevators>=1){
|
||||
// ret << "1"
|
||||
// }else{
|
||||
// ret << "0"
|
||||
// }
|
||||
ret << ""
|
||||
ret << ""
|
||||
// if(obj.elevators && obj.elevators>=1) {
|
||||
// ret << obj.elevators//µçÌÝÊý
|
||||
// }else{
|
||||
// ret << ""
|
||||
// }
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << "1"
|
||||
rets << ret
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << "1"
|
||||
rets << ret
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -367,7 +367,7 @@ class CcbCommonUtils {
|
||||
q.put("visible",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name","特房锦绣一方")
|
||||
// q.put("name","辽源一村")
|
||||
// q.put("name_cat", 0);
|
||||
// q.put("estate_type2", new BasicDBObject('$in', ["ĆŐͨץՏ","ÉĚץ"]))
|
||||
|
||||
|
||||
+6
@@ -57,6 +57,12 @@ class CcbFtpUtils {
|
||||
ftpUpload(dataCodeFileName)//上传加密码文件到FTP
|
||||
}
|
||||
|
||||
static def listFileNames(month){
|
||||
//上传到ftp
|
||||
sftpChannel = reConnect();
|
||||
Vector files = sftp.listFiles(ftpPath+ month+"/", sftpChannel);
|
||||
return files
|
||||
}
|
||||
|
||||
static main(args) {
|
||||
|
||||
|
||||
+9
-9
@@ -25,8 +25,8 @@ class DealPriceDataProcess {
|
||||
def task = new DealPriceDataProcess()
|
||||
// task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据
|
||||
// task.addLianJiaPrice();
|
||||
// task.deleteThirdPartyData()
|
||||
task.delDealByPe()
|
||||
task.deleteThirdPartyData()
|
||||
// task.delDealByPe()
|
||||
}
|
||||
|
||||
def delDealByPe(){
|
||||
@@ -370,26 +370,26 @@ class DealPriceDataProcess {
|
||||
|
||||
|
||||
def deleteThirdPartyData(){
|
||||
def citys = ["东莞"] //,"杭州""广州",
|
||||
def citys = CcbCommonUtils.citys
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def publishTime = '2021-10-01'
|
||||
def month = 202110
|
||||
def month = 202203
|
||||
def rows = []
|
||||
for(def city : citys) {
|
||||
def rets = findPeGteThirdParty(city, 202111)
|
||||
def rets = findPeGteThirdParty(city, 202204)
|
||||
def total = 0
|
||||
for(def m : rets){
|
||||
def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
|
||||
sqlInstance.executeUpdate(sql)
|
||||
sqlInstance236.executeUpdate(sql)
|
||||
}
|
||||
println city +" "+total
|
||||
println city +" "+rets.size()
|
||||
}
|
||||
}
|
||||
|
||||
def findPeGteThirdParty(cityName, yearmonth){
|
||||
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>20"
|
||||
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='µÚÈý·½' and cityname='${cityName}' and pe>50"
|
||||
def rets = []
|
||||
sqlInstance2.eachRow(sql,{r->
|
||||
sqlInstance236.eachRow(sql,{r->
|
||||
def map = [:]
|
||||
map.commid = r.commid
|
||||
map.price = r.target_avg_price
|
||||
|
||||
+2
-2
@@ -38,8 +38,8 @@ class ExportOracleData {
|
||||
exportCommunityBuileType()
|
||||
|
||||
exportThirdPartyPrice2(lastMonth)
|
||||
exportBigdataForsale("2022-02-19")
|
||||
exportBigdataDeal("2022-02-01")
|
||||
exportBigdataForsale("2022-03-19")
|
||||
exportBigdataDeal("2022-03-01")
|
||||
|
||||
|
||||
//计算结果相关
|
||||
|
||||
+23
@@ -3,8 +3,10 @@ package com.fangjia.script.ccbevaluation
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.jcraft.jsch.ChannelSftp
|
||||
import groovy.sql.Sql
|
||||
import com.fangjia.script.ccbevaluation.ronghe.SSHHelper
|
||||
import org.apache.commons.codec.digest.DigestUtils
|
||||
|
||||
class ExportOracleEvaluationData {
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
@@ -33,6 +35,8 @@ class ExportOracleEvaluationData {
|
||||
exportCommCaseDetailGj(curMonth)
|
||||
|
||||
def curDate = curMonth.toString()+"28"
|
||||
// 生成MD5文件
|
||||
exportMd5()
|
||||
SSHHelper jschCommand = new SSHHelper();
|
||||
System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
|
||||
// okÎļþ
|
||||
@@ -413,4 +417,23 @@ class ExportOracleEvaluationData {
|
||||
CcbFtpUtils.ftpUpload(null, dataCodeFileName, curDate)
|
||||
}
|
||||
|
||||
def exportMd5(){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
Vector files = CcbFtpUtils.listFileNames(curDate)
|
||||
Iterator fileVec = files.iterator()
|
||||
def fileRows = []
|
||||
while(fileVec.hasNext()){
|
||||
ChannelSftp.LsEntry file = fileVec.next()
|
||||
if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){
|
||||
fileRows << file.getFilename() +":"+ DigestUtils.md5Hex(file.getFilename())
|
||||
}
|
||||
}
|
||||
def fileName = "/ccb/md5.txt"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\房价网\\md5.txt"
|
||||
}
|
||||
FileUtils.createTxtFile(fileRows, fileName, "utf-8", false);
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "md5.txt 上传Ftp完成......"
|
||||
}
|
||||
}
|
||||
|
||||
+3
-3
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ThirdPartyPriceBatch extends BaseBatch {
|
||||
|
||||
def sDate= 202202//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def startDate = 202112
|
||||
def endDate = 202202
|
||||
def sDate= 202203//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def startDate = 202201
|
||||
def endDate = 202203
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
+8
-8
@@ -75,13 +75,13 @@ class TradeStatLog {
|
||||
def run(){
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rets = []
|
||||
rets << ["城市","12月总量","1月总量","2月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
rets << ["城市","1月总量","2月总量","3月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
|
||||
//def startDate = "2021-05-01"
|
||||
def startDate = "2021-12-01"
|
||||
def date = "2022-01-01"
|
||||
def endDate = "2022-02-01"
|
||||
def newDate = "2022-03-01"
|
||||
def startDate = "2022-01-01"
|
||||
def date = "2022-02-01"
|
||||
def endDate = "2022-03-01"
|
||||
def newDate = "2022-04-01"
|
||||
|
||||
for(def city : citys){
|
||||
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
@@ -131,7 +131,7 @@ class TradeStatLog {
|
||||
}
|
||||
|
||||
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-02成交统计.xlsx", rets, "数据")
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-03成交统计.xlsx", rets, "数据")
|
||||
println "表格生成完成"
|
||||
}
|
||||
|
||||
@@ -481,8 +481,8 @@ class TradeStatLog {
|
||||
def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"]
|
||||
def rets = []
|
||||
rets << headers
|
||||
def startDate = "2022-02-01"
|
||||
def endDate = "2022-03-01"
|
||||
def startDate = "2022-03-01"
|
||||
def endDate = "2022-04-01"
|
||||
for(def city : citys){
|
||||
|
||||
def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
|
||||
|
||||
+294
-2
@@ -2,25 +2,243 @@ package com.fangjia.script.ccbevaluation.data
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
|
||||
class CcbDistrictToDB {
|
||||
|
||||
static main(args) {
|
||||
def task = new CcbDistrictToDB();
|
||||
// task.run()
|
||||
task.run1()
|
||||
// task.run1()
|
||||
// task.meituanRun()
|
||||
// task.meituanSqrentRun()
|
||||
// task.updateDistrict()
|
||||
task.updateDongguangRegion()
|
||||
}
|
||||
|
||||
def updateDongguangRegion(){
|
||||
def cDate = DateUtils.parseFromTightByPattern("2022-04-27","yyyy-MM-dd")
|
||||
def dist = CommonDatautils.findAllDistrictByCity("东莞", "district", ["city","region","name","c_date","estate_type2"])
|
||||
for(def d : dist){
|
||||
def t = d.c_date
|
||||
if(t.getTime()>cDate.getTime()){
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def updateDistrict(){
|
||||
def citys = ["惠州","东莞","金华","苏州","桂林","潮州","廊坊","临沧","通辽","雅安"]
|
||||
def cDate = DateUtils.parseFromTightByPattern("2022-04-27","yyyy-MM-dd")
|
||||
def index = 0
|
||||
for(def city : citys){
|
||||
def dist = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","c_date","estate_type2"])
|
||||
for(def d : dist){
|
||||
def t = d.c_date
|
||||
if(t.getTime()>cDate.getTime()){
|
||||
if(d.name.contains("(")){
|
||||
def delstr = d.name.substring(d.name.indexOf("("), d.name.indexOf(")")+1)
|
||||
def newName = d.name.replace(delstr,"")
|
||||
index++
|
||||
println index+" "+d.city+" "+d.name+" "+newName
|
||||
def obj = CommonDatautils.findDistrict(d.city, d.region, newName, "district")
|
||||
if(!obj){
|
||||
println "update"
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("name", newName)
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
|
||||
d.name = newName
|
||||
addPrice(d)
|
||||
|
||||
def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1));
|
||||
CommonUtils.recalculationSqRent(d.city, d.region, newName, d.estate_type2, lastMonthDay,false,0)
|
||||
}else{
|
||||
println "delete"
|
||||
DbUtils.seaweed.remove(new BasicDBObject("_id",d._id))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def meituanSqrentRun(){
|
||||
ExcelUtils.modify("D:\\数据\\美团\\美团需入库小区合并后.xlsx", { d ->
|
||||
def city = d['F']
|
||||
def region = d['G']
|
||||
def newRegion = RegionData.formatRegion(city, region)
|
||||
if(newRegion) {
|
||||
def name = d['H']
|
||||
name = nameFileter(name)
|
||||
if(name){
|
||||
def estateType2 = "普通住宅"
|
||||
if(name.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(name.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}else{
|
||||
def type = d['I']
|
||||
if(type){
|
||||
if(type.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(type.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1));
|
||||
CommonUtils.recalculationSqRent(city, region, name, estateType2, lastMonthDay,false,0)
|
||||
}
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
def meituanRun(){
|
||||
def index = 0//美团需入库小区合并后.xlsx
|
||||
def keys = new CopyOnWriteArrayList()
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\美团\\美团需入库小区合并后.xlsx", { d ->
|
||||
def city = d['F']
|
||||
def region = d['G']
|
||||
def newRegion = RegionData.formatRegion(city, region)
|
||||
if(newRegion){
|
||||
def name = d['H']
|
||||
name = nameFileter(name)
|
||||
|
||||
def formatAddress = null
|
||||
if(d['E']){
|
||||
formatAddress = AddressUtil.formatAddress(city, d['E'])
|
||||
}
|
||||
|
||||
if(!formatAddress || (formatAddress && !formatAddress.contains("号") && !formatAddress.contains("交叉"))){
|
||||
def rets = OpenApiUtils.getGaoDeSearchLocation(city, region, name)
|
||||
def map = findStandardObject2(city,region, name, d['E'], rets);
|
||||
if(map && map.address){
|
||||
formatAddress = AddressUtil.formatAddress(city, map.address)
|
||||
if(!formatAddress){
|
||||
if(map.address.contains("交叉口")){
|
||||
formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉口")+3)
|
||||
}else if(map.address.contains("交叉")){
|
||||
formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉")+2)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Long glat = null
|
||||
Long glng = null
|
||||
Long blat = null
|
||||
Long blng = null
|
||||
if(d['K']){
|
||||
glat = d['L'].toString().toDouble() * 10000000000L
|
||||
glng = d['K'].toString().toDouble() * 10000000000L
|
||||
def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3)
|
||||
blng = baidu.x * 10000000000L
|
||||
blat = baidu.y * 10000000000L
|
||||
}
|
||||
|
||||
def estateType2 = "普通住宅"
|
||||
if(name.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(name.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}else{
|
||||
def type = d['I']
|
||||
if(type){
|
||||
if(type.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(type.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def key = city+"##"+newRegion+"##"+name
|
||||
if(name && !keys.contains(key)){
|
||||
def dist = CommonDatautils.findDistrict(city, newRegion, name, "district")
|
||||
if(!dist){
|
||||
keys << key
|
||||
def data = new BasicDBObject();
|
||||
data.put("name_cat",0)
|
||||
data.put("status",0)
|
||||
data.put("cat","district")
|
||||
data.put("c_date", new Date())
|
||||
data.put("m_date", new Date())
|
||||
data.put("city", city)
|
||||
data.put("region",newRegion)
|
||||
data.put("name", name)
|
||||
data.put("estate_type2",estateType2)
|
||||
data.put("address", formatAddress)
|
||||
if(glat){
|
||||
data.put("lat",glat)
|
||||
data.put("lng",glng)
|
||||
data.put("lat2",blat)
|
||||
data.put("lng2",blng)
|
||||
}
|
||||
def markMap = [:]
|
||||
markMap.put("data_source", "机器入库")
|
||||
data.put("mark_map",markMap)
|
||||
index++
|
||||
DbUtils.seaweed.save(data);
|
||||
|
||||
|
||||
println index+" "+data
|
||||
|
||||
def stop = false
|
||||
while (!stop){
|
||||
if(data._id.toString()){
|
||||
addPrice(data)
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println city +" "+region +" 区域无法标准化"
|
||||
}
|
||||
},0);
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
def addPrice(dist){
|
||||
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-03-01")
|
||||
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-25")
|
||||
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
|
||||
|
||||
for(def weekend : weekends){
|
||||
CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def run1(){
|
||||
def index = 0
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\新增小区数据_1236.xlsx", { d ->
|
||||
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\新增小区数据_1236.xlsx", { d ->
|
||||
def city = d['A']
|
||||
def region = d['B']
|
||||
def newRegion = RegionData.formatRegion(city, region)
|
||||
@@ -71,6 +289,7 @@ class CcbDistrictToDB {
|
||||
if(name){
|
||||
def dist = CommonDatautils.findDistrict(d['A'], newRegion, name, "district")
|
||||
if(!dist){
|
||||
|
||||
def data = new BasicDBObject();
|
||||
data.put("name_cat",0)
|
||||
data.put("status",0)
|
||||
@@ -221,4 +440,77 @@ class CcbDistrictToDB {
|
||||
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
|
||||
return name;
|
||||
}
|
||||
|
||||
|
||||
def findStandardObject2(def city, def region, def name, def address, def rets) {
|
||||
if (rets) {
|
||||
def bestObj = [:]
|
||||
if (rets.size() >= 1) {
|
||||
for (def mapObj : rets) {
|
||||
def score = 0
|
||||
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
|
||||
score = 0.2
|
||||
}
|
||||
if(mapObj.name && name){
|
||||
def newName = mapObj.name;
|
||||
def names = []
|
||||
if(mapObj.name && mapObj.name.indexOf("(")>-1){
|
||||
def name1 = mapObj.name.substring(0, mapObj.name.indexOf("("))
|
||||
names << name1
|
||||
def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")"))
|
||||
names << name2
|
||||
}else{
|
||||
names << newName
|
||||
}
|
||||
def maxNameScore = 0
|
||||
for(def n : names){
|
||||
double score1 = CosineSimilarAlgorithm.getSimilarity(n, name);
|
||||
if(score1 && score1>0){
|
||||
def nameScore = (score1 * 4)/10
|
||||
if(nameScore > maxNameScore){
|
||||
maxNameScore = nameScore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(maxNameScore){
|
||||
try{
|
||||
maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore)
|
||||
}catch(Exception e){
|
||||
maxNameScore = 0
|
||||
}
|
||||
|
||||
}
|
||||
score = maxNameScore + score;
|
||||
}
|
||||
|
||||
if(mapObj.address && address){
|
||||
double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address);
|
||||
if(addressScore && addressScore>0){
|
||||
try{
|
||||
addressScore = DataHandleUtils.getDoubleRound(2, addressScore)
|
||||
}catch(Exception e){
|
||||
addressScore = 0
|
||||
}
|
||||
score = addressScore*4/10 + score
|
||||
}
|
||||
}
|
||||
|
||||
if(!bestObj){
|
||||
mapObj.score = score
|
||||
bestObj = mapObj
|
||||
}else{
|
||||
if(score > bestObj.score){
|
||||
mapObj.score = score
|
||||
bestObj = mapObj
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return bestObj
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
+5
-3
@@ -16,7 +16,9 @@ import com.mongodb.DBObject
|
||||
import groovy.sql.Sql
|
||||
import org.apache.lucene.queryparser.classic.QueryParser
|
||||
|
||||
|
||||
/**
|
||||
* 重庆估值测试--2小时
|
||||
*/
|
||||
class EvaluationBatch {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
@@ -27,8 +29,8 @@ class EvaluationBatch {
|
||||
static main(args) {
|
||||
def task = new EvaluationBatch()
|
||||
// task.run()
|
||||
task.run1();
|
||||
// task.run2()
|
||||
// task.run1();
|
||||
task.run2()
|
||||
}
|
||||
|
||||
def run2(){
|
||||
|
||||
+32
-32
@@ -8,42 +8,17 @@ import com.fangjia.script.util.OpenApiClient
|
||||
class BlankMatch {
|
||||
static main(args) {
|
||||
def task = new BlankMatch()
|
||||
// task.run()
|
||||
task.run()
|
||||
// task.run1()
|
||||
// task.run2()
|
||||
task.run3()
|
||||
}
|
||||
|
||||
def run3(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row ->
|
||||
if(!row['C']){
|
||||
def dist = CommonDatautils.findDistrict("重庆", row['E'], row['F'], "district")
|
||||
if(dist){
|
||||
row['C'] = dist._id.toString()
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
def run2(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配不上(合并).xlsx",0,1)
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row ->
|
||||
if(!row['C']){
|
||||
for(def d : datas){
|
||||
if(d['B'] == row['B']){
|
||||
row['C'] = d['C']
|
||||
row['D'] = d['D']
|
||||
row['E'] = d['E']
|
||||
row['F'] = d['J']
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
// task.run3()
|
||||
}
|
||||
|
||||
def run() {
|
||||
//rMap = SeaweedDataMatchProcess.doMatch(city, region, d['G'], null);
|
||||
ExcelUtils.modify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row ->
|
||||
//def path = "D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx";
|
||||
def path = "D:\\数据\\重庆农商行\\2022-04-02\\测试.xlsx"
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['C']){
|
||||
def jsonObject = OpenApiClient.addressMatch("ÖŘÇě", row['B'], "property", false, null, null)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
@@ -54,8 +29,6 @@ class BlankMatch {
|
||||
row['G'] = jsonObject.get("address")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
});
|
||||
}
|
||||
|
||||
@@ -81,4 +54,31 @@ class BlankMatch {
|
||||
// }
|
||||
});
|
||||
}
|
||||
|
||||
def run3(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row ->
|
||||
if(!row['C']){
|
||||
def dist = CommonDatautils.findDistrict("重庆", row['E'], row['F'], "district")
|
||||
if(dist){
|
||||
row['C'] = dist._id.toString()
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
def run2(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配不上(合并).xlsx",0,1)
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row ->
|
||||
if(!row['C']){
|
||||
for(def d : datas){
|
||||
if(d['B'] == row['B']){
|
||||
row['C'] = d['C']
|
||||
row['D'] = d['D']
|
||||
row['E'] = d['E']
|
||||
row['F'] = d['J']
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
}
|
||||
|
||||
+254
@@ -0,0 +1,254 @@
|
||||
package com.fangjia.script.chongqing
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.gj.paimai.DataUtil
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
|
||||
class MatchEvaluateRegionTest {
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx"
|
||||
def total = 1500
|
||||
|
||||
static main(args) {
|
||||
def task = new MatchEvaluateRegionTest()
|
||||
task.run()//小区匹配
|
||||
task.run1()//地图匹配
|
||||
// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除
|
||||
task.addPrice()
|
||||
task.addArroundPrice()
|
||||
task.addAuthor()
|
||||
}
|
||||
def run(){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['F']){
|
||||
def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], "property", false, null, null)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['F'] = jsonObject.get("id")
|
||||
row['G'] = jsonObject.get("city")
|
||||
row['H'] = jsonObject.get("district")
|
||||
row['I'] = jsonObject.get("name")
|
||||
row['J'] = jsonObject.get("address")
|
||||
row['K'] = jsonObject.get("propertyType")
|
||||
row['L'] = jsonObject.get("map")
|
||||
// row['K'] = jsonObject.get("lng")
|
||||
// row['L'] = jsonObject.get("lat")
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def run1(){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['F']){
|
||||
def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], null, true, null, null)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['F'] = jsonObject.get("id")
|
||||
row['G'] = jsonObject.get("city")
|
||||
row['H'] = jsonObject.get("district")
|
||||
row['I'] = jsonObject.get("name")
|
||||
row['J'] = jsonObject.get("address")
|
||||
row['K'] = jsonObject.get("propertyType")
|
||||
row['L'] = jsonObject.get("map")
|
||||
// row['K'] = jsonObject.get("lng")
|
||||
// row['L'] = jsonObject.get("lat")
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def addPrice(){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['L']){
|
||||
def priceObj = CommonDatautils.findDistrictPrice(row['G'], row['H'], row['I'])
|
||||
if(priceObj){
|
||||
row['M'] = priceObj.avg_price
|
||||
row['N']= (int)(priceObj.avg_price * row['E'])
|
||||
}
|
||||
|
||||
def avg_price = findAvgPrice(row['M'])
|
||||
if(row['M'] && avg_price){
|
||||
def weekendPrice = row['M'].toString().toDouble()
|
||||
def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
|
||||
row['O'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15)
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
def addArroundPrice(){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(row['A'] && !row['K']) {
|
||||
def arroundObj = FangjiaClientUtils.arroundPrice(row['A'], row['B'])
|
||||
if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
|
||||
def priceObj = arroundObj.result
|
||||
row['M'] = priceObj.unitPrice
|
||||
row['N'] = (int) (priceObj.unitPrice * row['C'])
|
||||
row['Q'] = "周边价格"
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def addAuthor(){
|
||||
def authors = ["丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"]
|
||||
def total = 747
|
||||
def personTotal = authors.size()
|
||||
def avg = Math.ceil(total/personTotal)
|
||||
|
||||
def index = 0
|
||||
def count = 0;
|
||||
ExcelUtils.modify(path, { row ->
|
||||
if(index < avg){
|
||||
row['P'] = authors[count]
|
||||
}
|
||||
index++
|
||||
if(index >= avg){
|
||||
count++
|
||||
index = 0
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
def addDistrict(){
|
||||
def path = "D:\\数据\\重庆农商行\\2022-05-06\\需新增小区.xlsx"
|
||||
def keys = new CopyOnWriteArrayList()
|
||||
def index = 0
|
||||
ExcelUtils.modify(path, { d->
|
||||
def flag = d['M']
|
||||
if(!flag){
|
||||
def city = d['E']
|
||||
def region = d['F']
|
||||
def newRegion = RegionData.formatRegion(city, region)
|
||||
if(newRegion){
|
||||
def name = d['G']
|
||||
name = nameFileter(name)
|
||||
|
||||
def formatAddress = null
|
||||
def address = d['H']
|
||||
if(address){
|
||||
formatAddress = AddressUtil.formatAddress(city, address)
|
||||
if(!formatAddress){
|
||||
if(address.contains("交叉口")){
|
||||
formatAddress = address.substring(0, address.toString().indexOf("交叉口")+3)
|
||||
}else if(address.contains("交叉")){
|
||||
formatAddress = address.substring(0, address.toString().indexOf("交叉")+2)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Long glat = null
|
||||
Long glng = null
|
||||
Long blat = null
|
||||
Long blng = null
|
||||
if(d['K']){
|
||||
glat = d['L'].toString().toDouble() * 10000000000L
|
||||
glng = d['K'].toString().toDouble() * 10000000000L
|
||||
def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3)
|
||||
blng = baidu.x * 10000000000L
|
||||
blat = baidu.y * 10000000000L
|
||||
}
|
||||
|
||||
def estateType2 = "普通住宅"
|
||||
if(name.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(name.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}else{
|
||||
def type = d['I']
|
||||
if(type){
|
||||
if(type.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(type.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def key = city+"##"+newRegion+"##"+name
|
||||
if(name && !keys.contains(key)){
|
||||
def dist = CommonDatautils.findDistrict(city, newRegion, name, "district")
|
||||
if(!dist){
|
||||
keys << key
|
||||
def data = new BasicDBObject();
|
||||
data.put("name_cat",0)
|
||||
data.put("status",0)
|
||||
data.put("cat","district")
|
||||
data.put("c_date", new Date())
|
||||
data.put("m_date", new Date())
|
||||
data.put("city", city)
|
||||
data.put("region",newRegion)
|
||||
data.put("name", name)
|
||||
data.put("estate_type2",estateType2)
|
||||
data.put("address", formatAddress)
|
||||
if(glat){
|
||||
data.put("lat",glat)
|
||||
data.put("lng",glng)
|
||||
data.put("lat2",blat)
|
||||
data.put("lng2",blng)
|
||||
}
|
||||
def markMap = [:]
|
||||
markMap.put("data_source", "机器入库")
|
||||
data.put("mark_map",markMap)
|
||||
index++
|
||||
DbUtils.seaweed.save(data);
|
||||
println index+" "+data
|
||||
addPrice(data)
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println city +" "+region +" 区域无法标准化"
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def addPrice(dist){
|
||||
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-03-01")
|
||||
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-25")
|
||||
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
|
||||
|
||||
for(def weekend : weekends){
|
||||
CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def nameFileter(def name){
|
||||
name = name.replace("·","").replace(".","").replace("|","")
|
||||
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
|
||||
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
|
||||
if(name.contains("(")){
|
||||
def delstr = name.substring(name.indexOf("("), name.indexOf(")")+1)
|
||||
name = name.replace(delstr,"")
|
||||
}
|
||||
return name;
|
||||
}
|
||||
|
||||
def findAvgPrice(def commid){
|
||||
def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2";
|
||||
def avg_price = null
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
avg_price = r.target_avg_price
|
||||
});
|
||||
return avg_price
|
||||
}
|
||||
}
|
||||
+280
@@ -0,0 +1,280 @@
|
||||
package com.fangjia.script.chongqing
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.gj.paimai.DataUtil
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
|
||||
class MatchEvaluateTest {
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
def path = "D:\\数据\\重庆农商行\\2022-05-06\\模拟_测试数据.xlsx"
|
||||
def total = 1500
|
||||
|
||||
static main(args) {
|
||||
def task = new MatchEvaluateTest()
|
||||
// task.run()//小区匹配
|
||||
// task.run1()//地图匹配
|
||||
//// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除
|
||||
// task.addPrice()
|
||||
// task.addArroundPrice()
|
||||
// task.addAuthor()
|
||||
|
||||
task.stat()
|
||||
}
|
||||
|
||||
def addArroundPrice(){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(row['A'] && !row['K']) {
|
||||
def arroundObj = FangjiaClientUtils.arroundPrice(row['A'], row['B'])
|
||||
if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
|
||||
def priceObj = arroundObj.result
|
||||
row['K'] = priceObj.unitPrice
|
||||
row['L'] = (int) (priceObj.unitPrice * row['C'])
|
||||
row['O'] = "周边价格"
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
def run(){
|
||||
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['D'] && row['B']){
|
||||
def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['D'] = jsonObject.get("id")
|
||||
row['E'] = jsonObject.get("city")
|
||||
row['F'] = jsonObject.get("district")
|
||||
row['G'] = jsonObject.get("name")
|
||||
row['H'] = jsonObject.get("address")
|
||||
row['I'] = jsonObject.get("propertyType")
|
||||
row['J'] = jsonObject.get("map")
|
||||
// row['K'] = jsonObject.get("lng")
|
||||
// row['L'] = jsonObject.get("lat")
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def run1(){
|
||||
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['D'] && row['B']){
|
||||
def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], null, true, null, null)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['D'] = jsonObject.get("id")
|
||||
row['E'] = jsonObject.get("city")
|
||||
row['F'] = jsonObject.get("district")
|
||||
row['G'] = jsonObject.get("name")
|
||||
row['H'] = jsonObject.get("address")
|
||||
row['I'] = jsonObject.get("propertyType")
|
||||
row['J'] = jsonObject.get("map")
|
||||
// row['K'] = jsonObject.get("lng")
|
||||
// row['L'] = jsonObject.get("lat")
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def addPrice(){
|
||||
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['J'] && row['E']){
|
||||
def priceObj = CommonDatautils.findDistrictPrice(row['E'], row['F'], row['G'])
|
||||
if(priceObj){
|
||||
row['K'] = priceObj.avg_price
|
||||
row['L']= (int)(priceObj.avg_price * row['C'])
|
||||
}
|
||||
|
||||
|
||||
def avg_price = findAvgPrice(row['D'])
|
||||
if(row['K'] && avg_price){
|
||||
def weekendPrice = row['K'].toString().toDouble()
|
||||
def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
|
||||
row['M'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15)
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
def addAuthor(){
|
||||
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
|
||||
def authors = ["丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"]
|
||||
def personTotal = authors.size()
|
||||
def avg = Math.ceil(total/personTotal)
|
||||
|
||||
def index = 0
|
||||
def count = 0;
|
||||
ExcelUtils.modify(path, { row ->
|
||||
if(row['B']){
|
||||
if(index < avg){
|
||||
row['N'] = authors[count]
|
||||
}
|
||||
index++
|
||||
if(index >= avg){
|
||||
count++
|
||||
index = 0
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
def addDistrict(){
|
||||
def path = "D:\\数据\\重庆农商行\\2022-05-06\\需新增小区.xlsx"
|
||||
def keys = new CopyOnWriteArrayList()
|
||||
def index = 0
|
||||
ExcelUtils.modify(path, { d->
|
||||
def flag = d['M']
|
||||
if(!flag){
|
||||
def city = d['E']
|
||||
def region = d['F']
|
||||
def newRegion = RegionData.formatRegion(city, region)
|
||||
if(newRegion){
|
||||
def name = d['G']
|
||||
name = nameFileter(name)
|
||||
|
||||
def formatAddress = null
|
||||
def address = d['H']
|
||||
if(address){
|
||||
formatAddress = AddressUtil.formatAddress(city, address)
|
||||
if(!formatAddress){
|
||||
if(address.contains("交叉口")){
|
||||
formatAddress = address.substring(0, address.toString().indexOf("交叉口")+3)
|
||||
}else if(address.contains("交叉")){
|
||||
formatAddress = address.substring(0, address.toString().indexOf("交叉")+2)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Long glat = null
|
||||
Long glng = null
|
||||
Long blat = null
|
||||
Long blng = null
|
||||
if(d['K']){
|
||||
glat = d['L'].toString().toDouble() * 10000000000L
|
||||
glng = d['K'].toString().toDouble() * 10000000000L
|
||||
def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3)
|
||||
blng = baidu.x * 10000000000L
|
||||
blat = baidu.y * 10000000000L
|
||||
}
|
||||
|
||||
def estateType2 = "普通住宅"
|
||||
if(name.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(name.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}else{
|
||||
def type = d['I']
|
||||
if(type){
|
||||
if(type.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(type.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def key = city+"##"+newRegion+"##"+name
|
||||
if(name && !keys.contains(key)){
|
||||
def dist = CommonDatautils.findDistrict(city, newRegion, name, "district")
|
||||
if(!dist){
|
||||
keys << key
|
||||
def data = new BasicDBObject();
|
||||
data.put("name_cat",0)
|
||||
data.put("status",0)
|
||||
data.put("cat","district")
|
||||
data.put("c_date", new Date())
|
||||
data.put("m_date", new Date())
|
||||
data.put("city", city)
|
||||
data.put("region",newRegion)
|
||||
data.put("name", name)
|
||||
data.put("estate_type2",estateType2)
|
||||
data.put("address", formatAddress)
|
||||
if(glat){
|
||||
data.put("lat",glat)
|
||||
data.put("lng",glng)
|
||||
data.put("lat2",blat)
|
||||
data.put("lng2",blng)
|
||||
}
|
||||
def markMap = [:]
|
||||
markMap.put("data_source", "机器入库")
|
||||
data.put("mark_map",markMap)
|
||||
index++
|
||||
DbUtils.seaweed.save(data);
|
||||
println index+" "+data
|
||||
addPrice(data)
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println city +" "+region +" 区域无法标准化"
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def addPrice(dist){
|
||||
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-03-01")
|
||||
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-25")
|
||||
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
|
||||
|
||||
for(def weekend : weekends){
|
||||
CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def nameFileter(def name){
|
||||
name = name.replace("·","").replace(".","").replace("|","")
|
||||
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
|
||||
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
|
||||
if(name.contains("(")){
|
||||
def delstr = name.substring(name.indexOf("("), name.indexOf(")")+1)
|
||||
name = name.replace(delstr,"")
|
||||
}
|
||||
return name;
|
||||
}
|
||||
|
||||
def findAvgPrice(def commid){
|
||||
def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2";
|
||||
def avg_price = null
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
avg_price = r.target_avg_price
|
||||
});
|
||||
return avg_price
|
||||
}
|
||||
|
||||
def stat(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\2022-05-06\\模拟_测试数据_合并后 (1).xlsx",0,1)
|
||||
ExcelUtils.multiThreadingModify(path, { d ->
|
||||
if(d['B']){
|
||||
for(def row : datas){
|
||||
if(d['B']==row['B']){
|
||||
if(row['K']){
|
||||
d['P'] = row['K']
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,510 @@
|
||||
package com.fangjia.script.chongqing
|
||||
|
||||
import com.fangjia.core.util.SourceNameUtil
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
|
||||
class PocTest {
|
||||
|
||||
static main(args) {
|
||||
def task = new PocTest();
|
||||
task.run();
|
||||
}
|
||||
|
||||
def run(){
|
||||
// match("D:\\数据\\重庆农商行\\poc\\test.xlsx")
|
||||
// addPrice("D:\\数据\\重庆农商行\\poc\\test.xlsx")
|
||||
// standard("D:\\数据\\重庆农商行\\poc\\test.xlsx")
|
||||
// addLoudongNum("D:\\数据\\重庆农商行\\poc\\test.xlsx")
|
||||
// addDetail("D:\\数据\\重庆农商行\\poc\\test.xlsx", "D:\\数据\\重庆农商行\\poc\\楼盘详情.xlsx")
|
||||
|
||||
// createLoudong("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\底层数据-楼栋-户.xlsx")
|
||||
// createPaimai("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\法拍案例.xlsx")
|
||||
// createHouse("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\案例.xlsx")
|
||||
// createDealPrice("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\案例.xlsx")
|
||||
createEvaluation("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\案例.xlsx")
|
||||
}
|
||||
|
||||
|
||||
def match(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['P'] && row['C']){
|
||||
def jsonObject = OpenApiClient.addressMatch(row['A'], row['C'], "property", false, row['B'], null)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['P'] = jsonObject.get("name")
|
||||
row['Q'] = jsonObject.get("id")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addPrice(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(row['Q']){
|
||||
def dist = CommonDatautils.findDistrictById(row['Q'])
|
||||
if(dist){
|
||||
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
|
||||
if(priceObj){
|
||||
row['E'] = priceObj.avg_price
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def standard(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(row['C']){
|
||||
def jsonObject = OpenApiClient.standard2Action(row['A'], row['C'])
|
||||
if(jsonObject){
|
||||
println jsonObject
|
||||
row['F'] = jsonObject.get("province")
|
||||
row['G'] = jsonObject.get("city")
|
||||
row['H'] = jsonObject.get("district")
|
||||
row['I'] = jsonObject.get("township")
|
||||
row['J'] = jsonObject.get("street")
|
||||
row['K'] = jsonObject.get("streetNumber")
|
||||
row['L'] = jsonObject.get("buildingNumber")
|
||||
row['M'] = jsonObject.get("unitNumber")
|
||||
row['N'] = jsonObject.get("floor")
|
||||
row['O'] = jsonObject.get("roomNumber")
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
def addLoudongNum(path){
|
||||
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if (row['Q'] && row['L']) {
|
||||
def jsonObject = OpenApiClient.searchLoudong(row['Q'], row['L'], row['M'])
|
||||
if (jsonObject && jsonObject.result) {
|
||||
if(jsonObject.result.size()>=1){
|
||||
def obj = jsonObject.result[0]
|
||||
row['S'] = obj.id
|
||||
row['R'] = obj.houseNumber
|
||||
}
|
||||
}
|
||||
|
||||
if(row['O']){
|
||||
def jsonObject1 = OpenApiClient.searchHouseLoudong(row['Q'], row['L'], row['M'], row['O'])
|
||||
if (jsonObject1 && jsonObject1.result) {
|
||||
if(jsonObject.result.size()>=1){
|
||||
def obj = jsonObject.result[0]
|
||||
row['U'] = obj.id
|
||||
row['T'] = obj.roomNum
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def createLoudong(path, output){
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def datas = ExcelUtils.read(path, 0, 1)
|
||||
rets << ["输入城市","输入区域","输入地址","面积","楼栋号","楼栋ID","物业类型","主朝向","地上层数","地下层数","竣工日期","梯","户","总户数","景观","户号","户ID","主朝向","景观","户型结构"]
|
||||
for(def row : datas){
|
||||
def data = []
|
||||
data << row['A']
|
||||
data << row['B']
|
||||
data << row['C']
|
||||
data << row['D']
|
||||
if (row['Q'] && row['L']) {
|
||||
def jsonObject = OpenApiClient.searchLoudong(row['Q'], row['L'], row['M'])
|
||||
if (jsonObject && jsonObject.result && jsonObject.result.size()>=1) {
|
||||
def obj = jsonObject.result[0]
|
||||
data << obj.houseNumber
|
||||
data << obj.id
|
||||
data << obj.type// 物业类型
|
||||
data << ""//主朝向
|
||||
data << obj.floors//地上层数
|
||||
data << ""//地下层数
|
||||
data << obj.buildYear;//竣工日期
|
||||
data << ""//梯
|
||||
data << ""//户
|
||||
data << obj.households//总户数
|
||||
data << ""//景观
|
||||
}else{
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""// 物业类型
|
||||
data << ""//主朝向
|
||||
data << ""//地上层数
|
||||
data << ""//地下层数
|
||||
data << ""//竣工日期
|
||||
data << ""//梯
|
||||
data << ""//户
|
||||
data << ""//总户数
|
||||
data << ""//景观
|
||||
}
|
||||
|
||||
if(row['O']){
|
||||
def jsonObject1 = OpenApiClient.searchHouseLoudong(row['Q'], row['L'], row['M'], row['O'])
|
||||
if (jsonObject1 && jsonObject1.result) {
|
||||
if(jsonObject1.result.size()>=1){
|
||||
def obj = jsonObject1.result[0]
|
||||
|
||||
data << obj.roomNum//户号
|
||||
data << obj.id//户ID
|
||||
data << obj.direction//主
|
||||
data << obj.sight//景观
|
||||
if(obj.roomNum && obj.roomNum.contains("跃")){
|
||||
data << "跃式"//户型结构
|
||||
}else{
|
||||
data << ""//户型结构
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
rets << data
|
||||
}
|
||||
ExcelUtils.write(output, rets, "底层数据-楼栋-户")
|
||||
}
|
||||
|
||||
def addDetail(path, output){
|
||||
def datas = ExcelUtils.read(path, 0, 1)
|
||||
def rows = []
|
||||
rows << ["输入城市","输入区域","输入地址","小区名称","小区ID","小区别名","小区地址","板块","街道","基准价","经纬度","竣工时间","开发商名称","绿化率","容积率","车位个数","物业公司","物业费","住宅幢数","总户数","车位配比","占地面积","是否学区","周边教育","周边交通","周边商业","周边医院","周边金融机构"]
|
||||
for(def d : datas){
|
||||
if(d['Q']){
|
||||
def dist = CommonDatautils.findDistrictById(d['Q'])
|
||||
if(dist){
|
||||
def row = []
|
||||
def jsonObject = OpenApiClient.districtDetail(dist.city, dist.region, dist.name)
|
||||
if(jsonObject) {
|
||||
|
||||
row << d['A']
|
||||
row << d['B']
|
||||
row << d['C']
|
||||
row << jsonObject.get("name")//小区名称
|
||||
row << d['Q']//小区ID
|
||||
if(jsonObject.get("alias")){
|
||||
row << jsonObject.get("alias").join(",").replace('"',"")//小区别名
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
row << jsonObject.get("address")//小区地址
|
||||
row << jsonObject.get("commercialArea")//板块
|
||||
row << jsonObject.get("township")//街道***********
|
||||
row << jsonObject.get("avgPrice")//基准价
|
||||
row << jsonObject.get("lng").toString()+","+jsonObject.get("lat").toString()//经纬度
|
||||
row << jsonObject.get("finishBuildTime")//竣工时间
|
||||
row << jsonObject.get("developer")//开发商名称
|
||||
row << SeaweedUtils.formatVirescencePercent(jsonObject.get("greenPercentage"))//绿化率
|
||||
row << SeaweedUtils.formatCubagePercent(jsonObject.get("plotRatio"))//容积率
|
||||
row << SeaweedUtils.formatCarSiteCount(jsonObject.get("parking"))//车位个数
|
||||
row << jsonObject.get("propertyManagement")//物业公司
|
||||
row << jsonObject.get("managementFee")//物业费
|
||||
row << jsonObject.get("buildingCount")//住宅幢数
|
||||
row << SeaweedUtils.formatHouseholdCount(jsonObject.get("totalUnitCount"))//总户数
|
||||
row << SeaweedUtils.calcCarAndHouseholdRadio(jsonObject.get("parking"), jsonObject.get("totalUnitCount"))//车位配比*********
|
||||
row << SeaweedUtils.formatAllArea(jsonObject.get("totalUnitCount"))//占地面积
|
||||
|
||||
}
|
||||
|
||||
def facilityObj = OpenApiClient.facilitys(dist.city, dist.region, dist.name)
|
||||
if(facilityObj && facilityObj.result){
|
||||
def result = facilityObj.result
|
||||
if(result.keySchool){
|
||||
row << '是'
|
||||
}else {
|
||||
row << ""
|
||||
}
|
||||
|
||||
if(result.school){
|
||||
row << result.school.join(",").replace('"',"")
|
||||
}else {
|
||||
row << ""
|
||||
}
|
||||
|
||||
def bus = []
|
||||
if(result.bus){
|
||||
bus.addAll(result.bus)
|
||||
}
|
||||
if(result.metro){
|
||||
bus.addAll(result.metro)
|
||||
}
|
||||
|
||||
if(bus){
|
||||
row << bus.join(",").replace('"',"")
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
if(result.supermarket){
|
||||
row << result.supermarket.join(",").replace('"',"")
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
if(result.hospital){
|
||||
row << result.hospital.join(",").replace('"',"")
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
if(result.bank){
|
||||
row << result.bank.join(",").replace('"',"")
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
}
|
||||
rows << row
|
||||
}
|
||||
}
|
||||
}
|
||||
ExcelUtils.write(output, rows, "底层数据-小区")
|
||||
}
|
||||
|
||||
def createPaimai(path, output){
|
||||
def datas = ExcelUtils.read(path, 0, 1)
|
||||
def rows = []
|
||||
rows << ["输入城市","输入区域","输入地址","小区名称","标的名称","楼栋","户号","面积","案例时间","类型","状态","成交单价","评估单价"]
|
||||
def keys = []
|
||||
for(def d : datas){
|
||||
if(d['Q']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['Q'])
|
||||
if (dist) {
|
||||
def key = dist.city+"#"+dist.region+"#"+dist.name
|
||||
if(keys.contains(key)){
|
||||
continue
|
||||
}
|
||||
keys << key
|
||||
def jsonObject = OpenApiClient.searchPaimais(dist.city, dist.region, dist.name,10)
|
||||
if(jsonObject && jsonObject.result){
|
||||
def results = jsonObject.result
|
||||
for(def r: results){
|
||||
def row = []
|
||||
row << d['A']
|
||||
row << d['B']
|
||||
row << d['C']
|
||||
row << r.matchName
|
||||
row << r.address
|
||||
row << r.houseNum
|
||||
row << r.roomNum
|
||||
row << r.area
|
||||
row << r.startDate
|
||||
row << r.count
|
||||
row << r.status
|
||||
row << r.tradeAvgPrice
|
||||
row << r.tradePrice
|
||||
row << r.id
|
||||
rows << row
|
||||
}
|
||||
println dist.name +" "+results.size()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
ExcelUtils.write(output, rows, "拍卖案例")
|
||||
}
|
||||
|
||||
def createHouse(path, output){
|
||||
def datas = ExcelUtils.read(path, 0, 1)
|
||||
def rows = []
|
||||
rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
|
||||
def keys = []
|
||||
for(def d : datas) {
|
||||
if (d['Q']) {
|
||||
def key = d['Q']
|
||||
if (keys.contains(key)) {
|
||||
continue
|
||||
}
|
||||
keys << key
|
||||
def jsonObject = OpenApiClient.houseList(d['Q'],"20210501",20)
|
||||
if(jsonObject && jsonObject.result) {
|
||||
def results = jsonObject.result
|
||||
for (def r : results) {
|
||||
def row = []
|
||||
row << "挂牌"
|
||||
row << r.districtName
|
||||
if(r.estateType){
|
||||
row << r.estateType
|
||||
}else{
|
||||
if(r.purpose){
|
||||
row << r.purpose
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
}
|
||||
|
||||
if(r.room){
|
||||
def roomStr = r.room+"室"
|
||||
if(r.hall){
|
||||
roomStr = roomStr +r.hall+"厅"
|
||||
}
|
||||
row << roomStr
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
row << r.area
|
||||
row << r.fitment
|
||||
row << r.pDate
|
||||
row << ""
|
||||
row << r.avgPrice
|
||||
row << r.totalPrice
|
||||
if(r.floor){
|
||||
row << r.floor
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
if(r.height){
|
||||
row << r.height
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
row << r.direction
|
||||
row << SourceNameUtil.getResourceName(r.root)
|
||||
rows << row
|
||||
}
|
||||
|
||||
println d['Q'] +" "+results.size()
|
||||
}
|
||||
}
|
||||
}
|
||||
ExcelUtils.write(output, rows, "挂牌案例")
|
||||
}
|
||||
|
||||
def createEvaluation(path, output){
|
||||
def datas = ExcelUtils.read(path, 0, 1)
|
||||
def rows = []
|
||||
rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
|
||||
def keys = []
|
||||
for(def d : datas) {
|
||||
if (d['Q']) {
|
||||
def key = d['Q']
|
||||
if (keys.contains(key)) {
|
||||
continue
|
||||
}
|
||||
keys << key
|
||||
def jsonObject = OpenApiClient.evaluaionList(d['Q'],"20210501",20)
|
||||
if(jsonObject && jsonObject.result) {
|
||||
def results = jsonObject.result
|
||||
for (def r : results) {
|
||||
def row = []
|
||||
row << "评估"
|
||||
row << r.commName
|
||||
row << ""
|
||||
|
||||
if(r.roomcnt){
|
||||
def roomStr = r.roomcnt+"室"
|
||||
if(r.livingroomcnt){
|
||||
roomStr = roomStr +r.livingroomcnt+"厅"
|
||||
}
|
||||
row << roomStr
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
row << r.buildArea
|
||||
row << r.fitment
|
||||
row << r.cDate.substring(0, r.cDate.indexOf("T"))
|
||||
row << ""
|
||||
row << r.unitPrice
|
||||
row << r.totalPrice
|
||||
if(r.floor){
|
||||
row << r.floor
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
if(r.floorcnt){
|
||||
row << r.floorcnt
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
if(r.houseaspect && !r.houseaspect.contains("暂")){
|
||||
row << r.houseaspect
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
row << "房价网"
|
||||
rows << row
|
||||
}
|
||||
|
||||
println d['Q'] +" "+results.size()
|
||||
}
|
||||
}
|
||||
}
|
||||
ExcelUtils.write(output, rows, "评估案例")
|
||||
}
|
||||
|
||||
|
||||
def createDealPrice(path, output){
|
||||
def datas = ExcelUtils.read(path, 0, 1)
|
||||
def rows = []
|
||||
rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
|
||||
def keys = []
|
||||
for(def d : datas) {
|
||||
if (d['Q']) {
|
||||
def key = d['Q']
|
||||
if (keys.contains(key)) {
|
||||
continue
|
||||
}
|
||||
keys << key
|
||||
def jsonObject = OpenApiClient.searchDealPrice(d['Q'],"20210501",20)
|
||||
if(jsonObject && jsonObject.result) {
|
||||
def results = jsonObject.result
|
||||
for (def r : results) {
|
||||
def row = []
|
||||
row << "成交"
|
||||
row << r.districtName
|
||||
if(r.type){
|
||||
row << r.type
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
if(r.room){
|
||||
def roomStr = r.room+"室"
|
||||
if(r.hall){
|
||||
roomStr = roomStr +r.hall+"厅"
|
||||
}
|
||||
row << roomStr
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
row << r.area
|
||||
row << r.fitment
|
||||
row << r.tradeDate.substring(0, r.tradeDate.indexOf("T"))
|
||||
row << ""
|
||||
row << r.avgPrice
|
||||
row << r.totalPrice
|
||||
if(r.floor){
|
||||
row << r.floor
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
if(r.height){
|
||||
row << r.height
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
if(r.direction && !r.direction.contains("暂")){
|
||||
row << r.direction
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
row << r.source
|
||||
rows << row
|
||||
}
|
||||
|
||||
println d['Q'] +" "+results.size()
|
||||
}
|
||||
}
|
||||
}
|
||||
ExcelUtils.write(output, rows, "成交案例")
|
||||
}
|
||||
}
|
||||
+18
@@ -830,6 +830,7 @@ class CommonDatautils {
|
||||
keys.put("name_cat", 1);
|
||||
keys.put("finish_building_date", 1)
|
||||
keys.put("mapping_alias", 1)
|
||||
keys.put("sid", 1)
|
||||
DSCursor cursor = getDataStoreConnection().find(q, keys);
|
||||
if(cursor){
|
||||
for(def obj : cursor){
|
||||
@@ -3166,5 +3167,22 @@ class CommonDatautils {
|
||||
}
|
||||
return avg_price;
|
||||
}
|
||||
|
||||
static def findCityAvgRentRadio(def city, def weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("weekend", weekend)
|
||||
|
||||
def dbcursor = SQMETRE_RENT_COLL.find(q)
|
||||
def avg_price = null
|
||||
def radios = []
|
||||
for(def d : dbcursor){
|
||||
if(d.price_to_rent){
|
||||
radios << d.price_to_rent.toInteger()
|
||||
}
|
||||
}
|
||||
def avg_radios = (int)(radios.sum()/radios.size())
|
||||
return avg_radios;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+3
-3
@@ -16,9 +16,7 @@ import org.apache.commons.lang.math.RandomUtils;
|
||||
|
||||
class DataHandleUtils {
|
||||
|
||||
static main(args){
|
||||
|
||||
}
|
||||
|
||||
|
||||
static def getStandardRegion(def regionMap, def region){
|
||||
def newRegion = null;
|
||||
@@ -94,6 +92,8 @@ class DataHandleUtils {
|
||||
BigDecimal deSource = new BigDecimal(dSource);
|
||||
return deSource.setScale(scale, BigDecimal.ROUND_HALF_UP).doubleValue();
|
||||
}
|
||||
|
||||
|
||||
|
||||
static def trimChar(String c) {
|
||||
try {
|
||||
|
||||
+1
-1
@@ -35,7 +35,7 @@ class DistrictAroundFacilitiesBatch extends BaseBatch {
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
def cities = ["北京", "上海"]
|
||||
def cities = ["北京", "上海"]//
|
||||
for (def city : cities) {
|
||||
def ret = CommonDatautils.getAllDistricts("district", city)
|
||||
// def ret = getAllDistrictBaseInfo("上海", '闵行', '万源城', "district");
|
||||
|
||||
+4
-4
@@ -42,17 +42,17 @@ class YouDaDataParser {
|
||||
static def new_queryUrl = "http://192.168.0.228:9200/new_deal_price_lansi/doc/_search"
|
||||
|
||||
|
||||
static def esf_targetUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/doc"
|
||||
static def esf_targetUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/_doc"
|
||||
static def esf_queryUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/doc/_search"
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2022-02-01"
|
||||
static def statDate = "2022-03-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
// task.esfParser();//友达二手房存入mongo
|
||||
task.esfParser();//友达二手房存入mongo
|
||||
// task.newhouseToSaveEs();//新房保存到es
|
||||
task.esfhouseToSaveEs();//二手房保存到es
|
||||
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
|
||||
@@ -285,7 +285,7 @@ class YouDaDataParser {
|
||||
|
||||
|
||||
def esfParser(){
|
||||
def files = ["D:\\数据\\友达\\2022年3月\\查询1.DBF"]
|
||||
def files = ["D:\\数据\\友达\\2022年4月\\查询1.DBF"]
|
||||
def datas = []
|
||||
for(def fileName : files){
|
||||
def rows = readDBF(fileName);
|
||||
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
package com.fangjia.script.gj.apiLog
|
||||
|
||||
import com.alibaba.excel.util.DateUtils
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.Calendar;
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ApiTimeOutAnalyBatch extends BaseBatch{
|
||||
|
||||
DBCollection LOG = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.42", 52002)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new ApiTimeOutAnalyBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def uriCountMap = findTimeOutUri()
|
||||
sendMessage(uriCountMap)
|
||||
}
|
||||
|
||||
def sendMessage(uriCountMap){
|
||||
def message = "【Api接口近5分钟超过5条请求时间超3秒提醒】\n超时接口:"
|
||||
def total = message.length()
|
||||
uriCountMap.each{uri,count->
|
||||
if(count>=5){
|
||||
message = message + uri +"\n"
|
||||
}
|
||||
}
|
||||
if(message.length()>total){
|
||||
println message
|
||||
def access_token = "1e25f620ea2be926b2441e4f30d67f37abdff2e14088b03b41b4082b11fc34ea"
|
||||
DingDingMessageUtil.sendTextMessage(access_token, message)
|
||||
}
|
||||
}
|
||||
|
||||
def findTimeOutUri(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("time", new BasicDBObject('$gte', 3000))
|
||||
def startTime = DateUtil.add(new Date(), Calendar.MINUTE, -5)
|
||||
q.put("c_date", new BasicDBObject('$gte', startTime))
|
||||
def dbcursor = LOG.find(q)
|
||||
def uriCountMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def count = uriCountMap.get(d.request_uri)
|
||||
if(!count){
|
||||
count = 0
|
||||
}
|
||||
uriCountMap.put(d.request_uri, count + 1)
|
||||
}
|
||||
return uriCountMap
|
||||
}
|
||||
}
|
||||
+111
@@ -0,0 +1,111 @@
|
||||
package com.fangjia.script.gj.apiLog
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import groovy.sql.Sql
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ApiUserCountAnalyBatch extends BaseBatch{
|
||||
|
||||
static final dbFangjia = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia?autoReconnect=true&characterEncoding=utf-8&autoReconnect=true", "sa", "G00jia", "com.mysql.jdbc.Driver");
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection AUTHORITY_COLL = MongoUtils.getCollectionByName2("fangjia", "authority.api_user_authority", lineUrl)
|
||||
DBCollection VISIT_COLL = MongoUtils.getCollectionByName2("business", "api_visit_count", lineUrl)
|
||||
|
||||
|
||||
static main(args) {
|
||||
runBatch(new ApiUserCountAnalyBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def rets = findAllUsers()
|
||||
sendMessage(rets)
|
||||
}
|
||||
|
||||
|
||||
def sendMessage(def rets){
|
||||
def message = "【Api接口使用提醒】\n"
|
||||
def total = message.length()
|
||||
for(def r : rets){
|
||||
def type = "正式帐户"
|
||||
if(r.type!=0){
|
||||
type = "测试帐户"
|
||||
}
|
||||
message = message + r.company+ " 用户ID:"+r.user_id+"(${type}),有 "+r.uris.size()+" 个接口使用率已超过80%,接口有:"+r.uris.join(",")+"\n"
|
||||
}
|
||||
if(message.length()>total){
|
||||
println message
|
||||
def access_token = "1e25f620ea2be926b2441e4f30d67f37abdff2e14088b03b41b4082b11fc34ea"
|
||||
DingDingMessageUtil.sendTextMessage(access_token, message)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def findAllUsers(){
|
||||
def sql = "select id,COMPANY,is_test from api_user where dead_time>='"+DateUtils.formatDateByPattern(new Date(),'yyyy-MM-dd')+"'"
|
||||
def rows = dbFangjia.rows(sql)
|
||||
def rets = []
|
||||
for(def row : rows){
|
||||
def obj = findResources(row.id)
|
||||
if(obj){
|
||||
def map = obj.total_count_map
|
||||
def uris = []
|
||||
map.each{k,v->
|
||||
//def visit_count = findVisitCount(row.id, k)
|
||||
def total_visit_count = findAllVisitCount(row.id, k)
|
||||
println row.id +" "+k+" "+v+" "+total_visit_count
|
||||
if(v && total_visit_count){
|
||||
def precent = v*0.8;
|
||||
if(total_visit_count>=precent){
|
||||
uris << k
|
||||
}
|
||||
}
|
||||
}
|
||||
if(uris){
|
||||
rets << ["user_id":row.id,"company":row.company,"uris":uris,"type":row.is_test]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findAllVisitCount(userId, uri){
|
||||
def q = new BasicDBObject()
|
||||
q.put("user_id", userId.toString())
|
||||
q.put("uri", uri)
|
||||
q.put("visit_date", new BasicDBObject('$lte',DateUtils.truncateDay(new Date())))
|
||||
def dbcursor = VISIT_COLL.find(q)
|
||||
def total = 0
|
||||
for(def obj : dbcursor){
|
||||
total = total +obj.visit_count
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
|
||||
def findVisitCount(userId, uri){
|
||||
def q = new BasicDBObject()
|
||||
q.put("user_id", userId.toString())
|
||||
q.put("uri", uri)
|
||||
q.put("visit_date", DateUtils.truncateDay(new Date()))
|
||||
def obj = VISIT_COLL.findOne(q)
|
||||
if(obj){
|
||||
return obj.visit_count
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def findResources(def userId){
|
||||
def q = new BasicDBObject()
|
||||
q.put("user_id", userId)
|
||||
return AUTHORITY_COLL.findOne(q)
|
||||
}
|
||||
}
|
||||
@@ -50,4 +50,32 @@ class FileUtils {
|
||||
}
|
||||
out.close();
|
||||
}
|
||||
|
||||
static def createTxtFile(ret, fileName, charset, isAppend){
|
||||
File file = new File(fileName);
|
||||
if (!file.getParentFile().exists()) {
|
||||
boolean result = file.getParentFile().mkdirs();
|
||||
if (!result) {
|
||||
System.out.println("´´½¨Ä¿Â¼Ê§°Ü");
|
||||
}
|
||||
}
|
||||
|
||||
OutputStream out = new FileOutputStream(fileName, isAppend);
|
||||
def sb = new StringBuffer();
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
def index = 0
|
||||
for(def row : ret){
|
||||
if(row){
|
||||
index++
|
||||
sb.append(row)
|
||||
sb.append("\n");
|
||||
}
|
||||
}
|
||||
if(sb.length()>0){
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
}
|
||||
out.close();
|
||||
}
|
||||
}
|
||||
|
||||
+2
-2
@@ -25,11 +25,11 @@ class DataMatch {
|
||||
static main(args) {
|
||||
def task = new DataMatch()
|
||||
// task.run1()
|
||||
task.run2()
|
||||
// task.run2()
|
||||
// task.run3()
|
||||
// println task.findAddress("²ıÖݽֵÀÌÄÏã·")
|
||||
|
||||
// task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx")
|
||||
task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx")
|
||||
}
|
||||
|
||||
def run1(){
|
||||
|
||||
+246
@@ -0,0 +1,246 @@
|
||||
package com.fangjia.script.gj.crawl
|
||||
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.jsoup.Jsoup
|
||||
import org.openqa.selenium.By
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.WebElement
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
|
||||
class AnjukeDetailCrawl {
|
||||
DBCollection detailColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_detail")
|
||||
static main(args) {
|
||||
def task = new AnjukeDetailCrawl();
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
def url = "https://chongqing.anjuke.com/community/?from=navigation"
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def regions = getRegions(driver, url)
|
||||
getAllRegions(driver, regions)
|
||||
|
||||
}
|
||||
|
||||
|
||||
def getAllRegions(driver, regions){
|
||||
for(def url : regions){
|
||||
driver.get(url)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def total = getTotal(doc)
|
||||
if(total) {
|
||||
if (total > 1250) {
|
||||
|
||||
} else {
|
||||
def links = getAllLinks(url, driver)
|
||||
parse(links,driver)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def getAllLinks(url, driver){
|
||||
def stop = false
|
||||
def urls = []
|
||||
while(!stop) {
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def details = JQueryUtils.select(".list-cell a.li-row", doc)
|
||||
|
||||
for (def d : details) {
|
||||
urls << d.attr("href")
|
||||
}
|
||||
def nextEle = JQueryUtils.select(".next", doc)
|
||||
if (nextEle) {
|
||||
driver.findElement(By.cssSelector(".next")).click()
|
||||
}else{
|
||||
stop = true
|
||||
}
|
||||
if(urls.size()>20){
|
||||
stop = true
|
||||
}
|
||||
}
|
||||
|
||||
return urls
|
||||
}
|
||||
|
||||
def getTotal(doc){
|
||||
|
||||
def totalText = JQueryUtils.selectText(".total-info", doc)
|
||||
if(totalText){
|
||||
def total = totalText.substring(totalText.indexOf("到")+1, totalText.indexOf("个")).trim()
|
||||
if(total && total.isNumber()){
|
||||
return total.toInteger()
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
def getRegions(driver, url){
|
||||
driver.get(url)
|
||||
Thread.sleep(3000)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
// def loginIfm = JQueryUtils.select("#iframeLoginIfm", doc)
|
||||
// if(loginIfm){
|
||||
// def href = loginIfm[0].attr("src")
|
||||
// driver.get(href)
|
||||
// driver.findElementById("pwdTab").click()
|
||||
// WebElement elemUsername = driver.findElement(By.id("pwdUserNameIpt")); //获取用户名框
|
||||
// WebElement elemPassword = driver.findElement(By.id("pwdIpt")); //获取密码框
|
||||
// WebElement btn = driver.findElement(By.cssSelector("input[class='submit-btn']")); //获取点击登陆按钮。
|
||||
// elemUsername.sendKeys("17740818975");
|
||||
// elemPassword.sendKeys("12qwaszx");
|
||||
// btn.submit(); // 点击行为;
|
||||
//
|
||||
// def regionEles = JQueryUtils.select(".region-parents li a", doc)
|
||||
// def urls = []
|
||||
// for(def e : regionEles){
|
||||
// urls << e.attr("href")
|
||||
// }
|
||||
// return urls;
|
||||
// }
|
||||
def regionEles = JQueryUtils.select(".region-parents li a", doc)
|
||||
def urls = []
|
||||
for(def e : regionEles){
|
||||
urls << e.attr("href")
|
||||
}
|
||||
return urls;
|
||||
}
|
||||
|
||||
def parse(def links, driver){
|
||||
def rets = []
|
||||
for(def link : links){
|
||||
def district = parsing(link, driver)
|
||||
if(district){
|
||||
rets << district
|
||||
|
||||
if (rets.size() % 100 == 0) {
|
||||
def add= saveData(rets)
|
||||
println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
rets = []
|
||||
}
|
||||
}
|
||||
}
|
||||
if (rets.size()> 0) {
|
||||
def add= saveData(rets)
|
||||
println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def saveData(def results){
|
||||
def add = 0
|
||||
for(def ret : results){
|
||||
def q = new BasicDBObject()
|
||||
q.put("url", ret.url)
|
||||
def dbo = detailColl.findOne(q);
|
||||
|
||||
DBObject obj = new BasicDBObject();
|
||||
ret.each{k,v->
|
||||
obj.put(k,v)
|
||||
}
|
||||
|
||||
if(dbo){
|
||||
obj.m_date = new Date()
|
||||
detailColl.update(q, new BasicDBObject('$set', obj))
|
||||
}else{
|
||||
add++
|
||||
obj.c_date = new Date()
|
||||
obj.m_date = new Date()
|
||||
try{
|
||||
detailColl.save(obj)
|
||||
}catch(Exception e){
|
||||
println obj.url
|
||||
}
|
||||
}
|
||||
}
|
||||
return add
|
||||
}
|
||||
|
||||
|
||||
|
||||
def parsing(url, driver){
|
||||
|
||||
def district = [:]
|
||||
def stop = false
|
||||
def count = 0
|
||||
while(!stop){
|
||||
driver.get(url)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def headers = JQueryUtils.select(".crumbs a", doc)
|
||||
if(headers){
|
||||
if (headers && headers.size() >= 3) {
|
||||
district.city = headers[1].text().replace("小区", "")
|
||||
district.region = headers[2].text().replace("小区", "")
|
||||
district.name = headers[headers.size() - 1].text().replace("二手房", "")
|
||||
}
|
||||
|
||||
|
||||
district.address = JQueryUtils.selectText("div.column-1:contains(小区地址) .value", doc)
|
||||
|
||||
def priceStr = JQueryUtils.selectText(".average", doc)
|
||||
|
||||
if (priceStr && priceStr.isNumber()) {
|
||||
district.avg_price = priceStr.toInteger()
|
||||
}
|
||||
|
||||
def infoDiv = JQueryUtils.select(".info-list", doc)
|
||||
district.estate_type = getValue(infoDiv.select(".column-2:contains(物业类型) .value").text())
|
||||
district.estate_charge = getValue(infoDiv.select(".column-2:contains(物业费) .value").text())
|
||||
district.estate_company = getValue(infoDiv.select(".column-1:contains(物业公司) .value").text())
|
||||
district.developer = getValue(infoDiv.select(".column-1:contains(开发商) .value").text())
|
||||
|
||||
def houseCountStr = infoDiv.select(".column-2:contains(总户数) .value").text()
|
||||
if (houseCountStr) {
|
||||
houseCountStr = houseCountStr.replace("户", "")
|
||||
if (houseCountStr) {
|
||||
district.house_count = houseCountStr.toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
district.finish_date = getValue(infoDiv.select(".column-2:contains(竣工时间) .value").text())
|
||||
district.build_area = getValue(infoDiv.select(".column-2:contains(总建面积) .value").text())
|
||||
def carSiteStr = getValue(infoDiv.select(".column-2:contains(停车位) .value").text())
|
||||
if(carSiteStr){
|
||||
if(carSiteStr.contains("(")){
|
||||
carSiteStr = carSiteStr.substring(0, carSiteStr.indexOf("("))
|
||||
}
|
||||
if(carSiteStr && carSiteStr.isNumber()){
|
||||
district.car_site = carSiteStr
|
||||
}
|
||||
}
|
||||
district.volumetric_rate = getValue(infoDiv.select(".column-2:contains(容积率) .value").text())
|
||||
district.greening_rate = getValue(infoDiv.select(".column-2:contains(绿化率:) .value").text())
|
||||
district.build_type = getValue(infoDiv.select(".column-2:contains(建筑类型) .value").text())
|
||||
district.property_life = getValue(infoDiv.select(".column-2:contains(产权年限) .value").text())
|
||||
district.car_site_charge = getValue(infoDiv.select(".column-1:contains(停车费) .value").text())
|
||||
district.url = url
|
||||
district.c_date = new Date()
|
||||
district.m_date = new Date()
|
||||
district.root = "anjuke.com"
|
||||
|
||||
stop = true;
|
||||
}else{
|
||||
Thread.sleep(3000)
|
||||
count++
|
||||
}
|
||||
if(count>10){
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
|
||||
return district
|
||||
}
|
||||
|
||||
String getValue(String s){
|
||||
if(s && s.contains("暂无")){
|
||||
return null
|
||||
}
|
||||
return s
|
||||
}
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
package com.fangjia.script.gj.crawl
|
||||
|
||||
import org.jsoup.Jsoup
|
||||
|
||||
class CommonCrawl {
|
||||
static def getDocument(url, driver){
|
||||
def stop = false
|
||||
def error = 0
|
||||
def doc = null
|
||||
while(!stop){
|
||||
try{
|
||||
driver.get(url)
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
if(doc){
|
||||
stop = true;
|
||||
}else{
|
||||
error++
|
||||
Thread.sleep(5000)
|
||||
}
|
||||
}catch(Exception e){
|
||||
error++
|
||||
}
|
||||
if(error>=10){
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return doc
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -17,7 +17,7 @@ class TempMatchDistrict {
|
||||
}
|
||||
|
||||
def stat(){
|
||||
def citys = ["哈尔滨","齐齐哈尔","大庆","牡丹江","佳木斯","绥化","鸡西","七台河","双鸭山","鹤岗","黑河","伊春","大兴安岭"]
|
||||
def citys = ["广州","深圳","佛山","东莞","中山","湛江","惠州","江门","珠海","汕头","揭阳","茂名","清远","肇庆","潮州","河源","韶关","阳江","梅州","云浮","汕尾"]
|
||||
def total = 0
|
||||
for(def city : citys){
|
||||
def count = CommonDatautils.findAllDistrictCount("district", city)
|
||||
|
||||
+17
@@ -83,6 +83,23 @@ class FangjiaClientUtils {
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
public static def arroundPrice(city,address){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
try {
|
||||
address = address.replace("{","")
|
||||
// def url = "http://192.168.0.19:39080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8")
|
||||
def url = "http://192.168.0.100:38080/FangJiaDataService/evaluate?address="+URLEncoder.encode(address,"UTF-8")+"&city="+URLEncoder.encode(city,"UTF-8")+"&maxNum=10&radius=1000&size=100"
|
||||
// def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8")
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
} catch (Exception e) {
|
||||
println address +" "+content
|
||||
e.printStackTrace();
|
||||
}
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
|
||||
public static Map<String, Object> JsonToMap(JSONObject json) {
|
||||
Map<String, Object> columnValMap = new HashMap<String, Object>();
|
||||
|
||||
+15
-4
@@ -58,6 +58,8 @@ class DistrictIndexBatch extends BaseBatch{
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static def month = DateUtils.formatDateByPattern(new Date(),"yyyy-MM")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new DistrictIndexBatch(), args);
|
||||
}
|
||||
@@ -71,7 +73,7 @@ class DistrictIndexBatch extends BaseBatch{
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def citys = CommonDatautils.findAllCity();
|
||||
for(def city : citys){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2"])
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2","household_count"])
|
||||
def code = cityCodeDatas.get(city)
|
||||
def city_avg_total_price = null
|
||||
if(city!="ÉϺ£"){
|
||||
@@ -84,7 +86,7 @@ class DistrictIndexBatch extends BaseBatch{
|
||||
city_avg_total_price = 531.2615913
|
||||
}
|
||||
if(city_avg_total_price){
|
||||
calcute(city, dists, code, city_avg_total_price)
|
||||
// calcute(city, dists, code, city_avg_total_price)
|
||||
calcuteIndex(city, dists, code, city_avg_total_price)
|
||||
|
||||
}
|
||||
@@ -131,7 +133,7 @@ class DistrictIndexBatch extends BaseBatch{
|
||||
map.total_price = total_price
|
||||
//map.comp_value = potentiality
|
||||
//map.avg_total_price = total_price_index
|
||||
map.month = "2022-03"
|
||||
map.month = month
|
||||
map.c_date = DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
|
||||
|
||||
//datas << [d.city, d.region, d.name, total_price_index, avg_charge_index, rank_index, avg_cost_index,potentiality]
|
||||
@@ -229,7 +231,7 @@ class DistrictIndexBatch extends BaseBatch{
|
||||
|
||||
map.comp_value = potentiality
|
||||
map.avg_total_price = total_price_index
|
||||
map.month = "2022-03"
|
||||
map.month = month
|
||||
map.m_date = DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
|
||||
|
||||
map.finish_date = year
|
||||
@@ -240,6 +242,15 @@ class DistrictIndexBatch extends BaseBatch{
|
||||
if(rMap.sell_avg_price && sqmetre_rent){
|
||||
map.price_to_rent = (rMap.sell_avg_price/sqmetre_rent).toInteger()
|
||||
}
|
||||
if(d.household_count){
|
||||
map.household_count = d.household_count
|
||||
}
|
||||
if(d.lat){
|
||||
def lMap = [:]
|
||||
lMap.lon = d.lng/10000000000.0
|
||||
lMap.lat = d.lat/10000000000.0
|
||||
map.location = lMap
|
||||
}
|
||||
def newUrl = targetUrl +"_doc/"+map.id
|
||||
httpSaveData(newUrl, map)
|
||||
|
||||
|
||||
+354
@@ -0,0 +1,354 @@
|
||||
package com.fangjia.script.gj.meituan
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.gj.paimai.DataUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.GeoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
class MatchTest {
|
||||
static main(args) {
|
||||
def task = new MatchTest()
|
||||
// task.run()
|
||||
// task.addNum()
|
||||
task.dongguang()
|
||||
// task.meituanDel()
|
||||
}
|
||||
|
||||
def meituanDel(){
|
||||
def path = "D:\\数据\\美团\\美团需入库小区合并后.xlsx"
|
||||
def index = 0
|
||||
ExcelUtils.modify(path, { row ->
|
||||
def flag = row['O']
|
||||
if(flag){
|
||||
def city = row['F']
|
||||
def region = row['G']
|
||||
def name = nameFileter(row['H'])
|
||||
def obj = null
|
||||
if(city == "东莞"){
|
||||
obj = CommonDatautils.findDistrict(city, null, name, "district")
|
||||
}else{
|
||||
obj = CommonDatautils.findDistrict(city, region, name, "district")
|
||||
}
|
||||
if(obj){
|
||||
index++
|
||||
println index +" "+ city +' '+ region+" "+name
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("name_cat", 11)
|
||||
udata.put("status", 1)
|
||||
udata.put("m_date",new Date())
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", new ObjectId(obj._id.toString()))
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
def dongguang(){
|
||||
def path = "D:\\数据\\美团\\美团东莞.xlsx"//美团金融小区测试list-匹配.xlsx
|
||||
ExcelUtils.modify(path, { row ->
|
||||
def region = null
|
||||
if(row['B'] && row['B']=="NULL" ){
|
||||
region = null
|
||||
}
|
||||
def name = nameFileter(row['H'])
|
||||
def address = row['E']
|
||||
if (name && name.contains("湖畔新家")) {
|
||||
|
||||
def obj = CommonDatautils.findDistrict("东莞", "东莞", name, "district")
|
||||
if(obj){
|
||||
def json = GaoDeApiUtils.geo(row['F'], address)
|
||||
if(json && json.count>=1){
|
||||
def m = json.geocodes[0]
|
||||
if(m){
|
||||
def twinshop = getStreetInfo(m.location)
|
||||
if(twinshop){
|
||||
def newRegion = RegionData.formatRegion("东莞",twinshop)
|
||||
if(newRegion){
|
||||
def dbObj = CommonDatautils.findDistrict("东莞", newRegion, name, "district")
|
||||
if(dbObj){
|
||||
println "delete"
|
||||
DbUtils.seaweed.remove(new BasicDBObject("_id",obj._id))
|
||||
}else{
|
||||
println "update"
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("region", newRegion)
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
// println address +" "+twinshop
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
|
||||
}
|
||||
|
||||
def getStreetInfo(def glocation){
|
||||
def township = null
|
||||
if(glocation){
|
||||
def jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||
if(jsonObject && jsonObject.regeocode) {
|
||||
def addressComponent = jsonObject.regeocode.addressComponent
|
||||
if(addressComponent){
|
||||
township = addressComponent.township
|
||||
}
|
||||
}
|
||||
}
|
||||
return township;
|
||||
}
|
||||
|
||||
def addNum(){
|
||||
def path = "D:\\数据\\美团\\美团金融小区测试list-所有数据.xlsx"//美团金融小区测试list-匹配.xlsx
|
||||
def index = 0
|
||||
ExcelUtils.modify(path, { row ->
|
||||
index++
|
||||
row['A'] = index
|
||||
},0)
|
||||
}
|
||||
|
||||
def run() {
|
||||
def citys = ["潮州","金华","惠州","东莞","苏州","桂林","廊坊","临沧","通辽","雅安"]//
|
||||
def weekend = DateUtils.getFirstDayOfWeek(new Date());
|
||||
def cityLevelMap = [:]
|
||||
def radioMap = [:]
|
||||
for(def city : citys){
|
||||
|
||||
// def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1));
|
||||
// def avg_radio = CommonDatautils.findCityAvgRentRadio(city, lastMonthDay)
|
||||
// radioMap.put(city, avg_radio)
|
||||
|
||||
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region","name"])
|
||||
def priceMap = CommonDatautils.queryDistrictPrice(city, weekend)
|
||||
def datas = addPriceData(dists, priceMap);
|
||||
datas = calcuteLevel(datas);
|
||||
cityLevelMap.put(city, ["price":priceMap, "level":datas])
|
||||
println city +" 楼盘评级同步完成"
|
||||
|
||||
}
|
||||
|
||||
|
||||
def path = "D:\\数据\\美团\\美团金融小区测试list-所有数据-评级.xlsx"//美团金融小区测试list-所有数据-评级.xlsx
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
def region = null
|
||||
if(row['C'] && row['C']=="NULL" ){
|
||||
region = null
|
||||
}
|
||||
def name = row['D']
|
||||
def address = row['F']
|
||||
def city = RegionData.formatCity(row['B'])
|
||||
|
||||
if(name && !row['N']){
|
||||
if(city){
|
||||
def map = cityLevelMap.get(city)
|
||||
def rMap = null
|
||||
if(map){
|
||||
rMap = getArroundDistrictPriceByAddress(city, address, map.level)
|
||||
}
|
||||
|
||||
if(rMap){
|
||||
if(rMap.rank){
|
||||
row['Q'] = rMap.rank
|
||||
}
|
||||
if(rMap.level){
|
||||
row['P'] = rMap.level
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (name && row['N']) {
|
||||
//// def jsonObject = OpenApiClient.addressMatch(row['A'], address, null, true, region, name)
|
||||
// def jsonObject = OpenApiClient.addressMatch(row['A'], name, null, true, region, null)
|
||||
// if (jsonObject != null && jsonObject.precise != -1) {
|
||||
// row['F'] = jsonObject.get("city")
|
||||
// row['G'] = jsonObject.get("district")
|
||||
// row['H'] = jsonObject.get("name")
|
||||
// row['I'] = jsonObject.get("propertyType")
|
||||
// row['J'] = jsonObject.get("map")
|
||||
// row['K'] = jsonObject.get("lng")
|
||||
// row['L'] = jsonObject.get("lat")
|
||||
// }
|
||||
|
||||
if(row['G'] && row['H'] && row['I']){
|
||||
// def priceObj = CommonDatautils.findDistrictPrice(row['F'], row['G'], row['H'])
|
||||
// if(priceObj){
|
||||
// row['M'] = priceObj.avg_price
|
||||
// }
|
||||
// def sqmetreRentPrice = CommonDatautils.findNewSqmetreRent(row['F'], row['G'], row['H'])
|
||||
// if(sqmetreRentPrice){
|
||||
// row['N'] = sqmetreRentPrice
|
||||
// }
|
||||
|
||||
if(row['N'] && !row['O']){
|
||||
def avg_rent_radio = radioMap.get(city)
|
||||
row['O'] = (int)(row['N']/avg_rent_radio)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def getArroundDistrictPriceByAddress(def city, def address, def levels){
|
||||
def json = GaoDeApiUtils.geo(city, address)
|
||||
def location = null
|
||||
def detail = new BasicDBObject()
|
||||
def level = null
|
||||
def rank = null
|
||||
if(json && json.count>=1){
|
||||
def m = json.geocodes[0]
|
||||
if(m){
|
||||
location = m.location
|
||||
level = m.level
|
||||
}
|
||||
}
|
||||
if(location){
|
||||
def arrs = location.split(",")
|
||||
detail.lng = arrs[0].toString().toDouble()*10000000000L
|
||||
detail.lat = arrs[1].toString().toDouble()*10000000000L
|
||||
}
|
||||
if(detail.lng){
|
||||
detail.city = city
|
||||
|
||||
List<DBObject> list = DbUtils.findAroundDistrictByRadius(detail, 3000)
|
||||
if(list && list.size()>10){
|
||||
list = sortByDistance(detail, list);
|
||||
list = list[0..9]
|
||||
}
|
||||
def arroundAvgPrice = getArroundAvgPrice(list)
|
||||
if(arroundAvgPrice){
|
||||
|
||||
def mod = Integer.MAX_VALUE;
|
||||
for(def d : levels){
|
||||
def newMod = java.lang.Math.abs( d.avg_price - arroundAvgPrice)
|
||||
if(newMod<mod){
|
||||
mod = newMod
|
||||
rank = d.level
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
def m = [:]
|
||||
m.level = level
|
||||
m.rank = rank
|
||||
return m
|
||||
}
|
||||
|
||||
|
||||
def sortByDistance(d, list){
|
||||
for(def nb : list){
|
||||
def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0)
|
||||
nb.distance = distance
|
||||
}
|
||||
//排序
|
||||
Collections.sort(list, new Comparator() {
|
||||
@Override
|
||||
int compare(Object o1, Object o2) {
|
||||
if(o1.distance > o2.distance)
|
||||
return 1;//第二个元素(o1)比第一个元素(o2)大,返回-1
|
||||
if(o1.distance == o2.distance)
|
||||
return 0;
|
||||
return -1;
|
||||
}
|
||||
})
|
||||
return list;
|
||||
}
|
||||
|
||||
def addPriceData(dists, priceMap){
|
||||
def newLists = []
|
||||
for(def d : dists){
|
||||
def price = priceMap.get(d.city+"##"+d.region+"##"+d.name);
|
||||
if(price){
|
||||
d.avg_price = price;
|
||||
newLists << d;
|
||||
}
|
||||
}
|
||||
if(newLists){
|
||||
newLists.sort {a,b->
|
||||
def v1 = a.avg_price
|
||||
def v2 = b.avg_price
|
||||
return v1.compareTo(v2)
|
||||
}
|
||||
}
|
||||
return newLists;
|
||||
}
|
||||
|
||||
def calcuteLevel(def datas){
|
||||
def total = datas.size()
|
||||
def v = (int)(total/10)
|
||||
def start = 0;
|
||||
def end = 0;
|
||||
def map = [:]
|
||||
for(int i=1;i<=10;i++){
|
||||
end = end + v;
|
||||
if(i == 10){
|
||||
map.put(i, [start, total])
|
||||
}else{
|
||||
map.put(i, [start, end])
|
||||
}
|
||||
start = end;
|
||||
}
|
||||
for(def d=0;d<datas.size();d++){
|
||||
def dist = datas[d]
|
||||
|
||||
for (def o in map) {
|
||||
def ranges = o.value
|
||||
def m = ranges[0]
|
||||
def n = ranges[1]
|
||||
if(d==0){
|
||||
if(d<=n){
|
||||
dist.level = 10 - o.key + 1
|
||||
break
|
||||
}
|
||||
}else{
|
||||
if(d>m && d<=n){
|
||||
dist.level = 10 - o.key + 1
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return datas;
|
||||
}
|
||||
|
||||
def getArroundAvgPrice(def list){
|
||||
def prices = []
|
||||
for(def l : list){
|
||||
def priceObj = CommonDatautils.findDistrictPrice(l.city, l.region, l.name)
|
||||
if(priceObj){
|
||||
prices << priceObj.avg_price
|
||||
}
|
||||
}
|
||||
if(prices){
|
||||
def avg_price = (prices.sum()/prices.size()).toInteger()
|
||||
return avg_price;
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def nameFileter(def name){
|
||||
name = name.replace("·","").replace(".","").replace("|","")
|
||||
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
|
||||
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
|
||||
return name;
|
||||
}
|
||||
|
||||
}
|
||||
+33
-13
@@ -45,9 +45,21 @@ public class BeiKeCrawlByChrome {
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def cityMap = getAllCityUrls(driver)
|
||||
def citys = getAllCityUrls(driver)
|
||||
citys = citys.reverse()
|
||||
def start = 0
|
||||
// for(def i=0;i<citys.size();i++){
|
||||
// def m = citys[i]
|
||||
// println i+" "+ m.city
|
||||
// if(m.city=="Ìì½ò"){
|
||||
// start = i
|
||||
// break;
|
||||
// }
|
||||
//
|
||||
// }
|
||||
// citys = citys[(start+1)..(citys.size()-1)]
|
||||
getLogin(driver)
|
||||
crawlData(driver, cityMap)
|
||||
crawlData(driver, citys)
|
||||
|
||||
|
||||
}
|
||||
@@ -85,13 +97,13 @@ public class BeiKeCrawlByChrome {
|
||||
}
|
||||
}
|
||||
|
||||
def crawlData(driver, cityMap){
|
||||
def crawlData(driver, citys){
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
cityMap.each{city, cityUrl->
|
||||
for(def m : citys){
|
||||
//def detailUrls = getPageUrls(driver, cityUrl)
|
||||
def detailUrls = getPageDeail(driver, city, cityUrl)
|
||||
println cityUrl +" ÏêÇéÁ´½Ó£º"+detailUrls.size()
|
||||
def detailUrls = getPageDeail(driver, m.city, m.url)
|
||||
println m.city +" ÏêÇéÁ´½Ó£º"+detailUrls.size()
|
||||
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
|
||||
}
|
||||
}
|
||||
@@ -141,7 +153,7 @@ public class BeiKeCrawlByChrome {
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
|
||||
if(dealDate && dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e)
|
||||
@@ -150,6 +162,10 @@ public class BeiKeCrawlByChrome {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if(page>=100){
|
||||
stop = true;
|
||||
break;
|
||||
}
|
||||
page++;
|
||||
}else{
|
||||
stop = true;
|
||||
@@ -182,7 +198,7 @@ public class BeiKeCrawlByChrome {
|
||||
}
|
||||
|
||||
def areaStr = arrs[2].replace("ƽÃ×", "")
|
||||
if (areaStr) {
|
||||
if (areaStr && areaStr.isNumber()) {
|
||||
detail.area = areaStr.toDouble()
|
||||
}
|
||||
|
||||
@@ -337,17 +353,21 @@ public class BeiKeCrawlByChrome {
|
||||
driver.get("https://www.ke.com/city/")
|
||||
Thread.sleep(8000)
|
||||
def cityDoc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
|
||||
def cityMap = [:]
|
||||
def cityEles = JQueryUtils.select(".city_list_ul .city_province a", cityDoc)
|
||||
def list = []
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
def href = e.attr("href")
|
||||
if(e.text().contains("¸£ÖÝ")){
|
||||
cityMap.put(e.text(), "https:"+href)
|
||||
if(e.text().contains("ÖØÇì")){
|
||||
def cityMap = [:]
|
||||
cityMap.city = e.text()
|
||||
cityMap.url = "https:"+href
|
||||
list << cityMap
|
||||
//cityMap.put(e.text(), "https:"+href)
|
||||
}
|
||||
}
|
||||
}
|
||||
return cityMap
|
||||
return list
|
||||
}
|
||||
|
||||
def getLogin(driver){
|
||||
|
||||
+18
-10
@@ -42,7 +42,6 @@ System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Loca
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
|
||||
|
||||
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def proviceMap = getDocument(driver);
|
||||
def citys = getCity(driver,proviceMap)
|
||||
@@ -59,9 +58,11 @@ def getDetailLinks(driver, cityList){
|
||||
citys.each{city,id->
|
||||
def urls = []
|
||||
if(!zxcitys.contains(cityMap.provice)){
|
||||
driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&city=${id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport=");
|
||||
def url = "https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&city=${id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport="
|
||||
getDocument(url, driver)
|
||||
}else{
|
||||
driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport=");
|
||||
def url = "https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport="
|
||||
getDocument(url, driver)
|
||||
}
|
||||
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
@@ -89,7 +90,7 @@ def getDetailLinks(driver, cityList){
|
||||
}
|
||||
}
|
||||
|
||||
def rets = batchParse(urls);
|
||||
def rets = batchParse(urls, driver);
|
||||
saveToDB(rets);
|
||||
println city +" "+urls.size()
|
||||
}
|
||||
@@ -124,7 +125,7 @@ def getCity(driver, proviceMap){
|
||||
for(def c : cityEles){
|
||||
def cityid = c.attr("data-cityid")
|
||||
def city = c.text()
|
||||
if(!["石家庄市","太原市","长治市","朔州市","廊坊市","衡水市","唐山市","秦皇岛市","邯郸市","邢台市","保定市","沧州市","张家口市","承德市"].contains(city)){
|
||||
if(!["石家庄市","太原市","齐齐哈尔市","哈尔滨市","鸡西市","长治市","朔州市","廊坊市","衡水市","唐山市","秦皇岛市","邯郸市","邢台市","保定市","沧州市","张家口市","承德市"].contains(city)){
|
||||
cityMap.put(city, cityid)
|
||||
}
|
||||
|
||||
@@ -149,7 +150,7 @@ def getCity(driver, proviceMap){
|
||||
|
||||
def getDocument(driver){
|
||||
|
||||
driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=9&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&canLoan=&isRestricted=");
|
||||
driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=1&canLoan=&isRestricted=&insuranceSupport=");
|
||||
WebDriverWait wait = new WebDriverWait(driver,20)
|
||||
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("lots-list"))));
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
@@ -159,7 +160,7 @@ def getDocument(driver){
|
||||
for(def ele : proviceEles){
|
||||
def provice = ele.text()
|
||||
def value = ele.attr("data-proviceid")
|
||||
if(value && provice!="北京市" && provice!="天津市"){
|
||||
if(value && provice!="北京市" && provice!="天津市" && provice!="吉林省" && provice!="辽宁省" && provice!="内蒙古自治区" && provice!="山西省" && provice!="河北省" && provice!="江苏省" && provice!="黑龙江省"){
|
||||
provinceMap.put(provice, value)
|
||||
}
|
||||
}
|
||||
@@ -167,8 +168,8 @@ def getDocument(driver){
|
||||
return provinceMap;
|
||||
}
|
||||
|
||||
def batchParse(def urls){
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def batchParse(def urls, driver){
|
||||
|
||||
def ret = []
|
||||
for(def map :urls){
|
||||
def doc = getDocument(map.url, driver)
|
||||
@@ -280,7 +281,7 @@ def parse(province, city, url,doc){
|
||||
result.lookCount = JQueryUtils.selectText(".look_count span", doc)
|
||||
|
||||
def startDateStr = JQueryUtils.selectText("span:contains(ÅÄÂô»á¶¨ÓÚ)", doc)
|
||||
if(startDateStr){
|
||||
if(startDateStr && startDateStr.endsWith("定于") && startDateStr.endsWith("日")){
|
||||
startDateStr = startDateStr.toString().substring(startDateStr.indexOf("¶¨ÓÚ")+2, startDateStr.indexOf("ÈÕ")+1).trim()
|
||||
}
|
||||
|
||||
@@ -329,6 +330,13 @@ def parse(province, city, url,doc){
|
||||
result.tradeAvgPrice = (int) (result.tradePrice / result.area)
|
||||
}
|
||||
|
||||
if(result.finishDate){
|
||||
result.finishDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.finishDate, "2000-01-01")
|
||||
}
|
||||
if(result.startDate){
|
||||
result.startDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.startDate, "2050-01-01")
|
||||
}
|
||||
|
||||
result.content = DataUtil.zip(doc.toString())
|
||||
result.introduce = JQueryUtils.select([".d-block table:eq(1)"], doc).toString()
|
||||
result.url = url
|
||||
|
||||
+23
-2
@@ -16,6 +16,27 @@ import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class DataUtil {
|
||||
|
||||
|
||||
static def checkMinDate(curDate, checkDate){
|
||||
if(curDate){
|
||||
def minDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(checkDate,"yyy-MM-dd")
|
||||
if(curDate.getTime()<minDate.getTime()){
|
||||
return null
|
||||
}
|
||||
}
|
||||
return curDate
|
||||
}
|
||||
|
||||
static def checkMaxDate(curDate, checkDate){
|
||||
if(curDate){
|
||||
def maxDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(checkDate,"yyy-MM-dd")
|
||||
if(curDate.getTime()>maxDate.getTime()){
|
||||
return null
|
||||
}
|
||||
}
|
||||
return curDate
|
||||
}
|
||||
static def getAreaNumber(def input, def maxArea) {
|
||||
Pattern pattern = Pattern.compile('\\d+(.\\d+)?(ƽ·½Ã×|©O|m2|·½Ã×|ƽ)')
|
||||
Matcher matcher = pattern.matcher(input)
|
||||
@@ -112,10 +133,10 @@ class DataUtil {
|
||||
if (list.size() >= 1) {
|
||||
String s1 = list.get(0)
|
||||
if (s1.contains("ÍòÔª")) {
|
||||
s1 = s1.toString().replace("起拍价:", "").replace("万元", "").replace(",", "").replace(",", "")
|
||||
s1 = s1.toString().replace("起拍价:", "").replace("万元", "").replace(",", "").replace(",", "").replace(" ","")
|
||||
return (int) (s1.toDouble() * 10000)
|
||||
} else {
|
||||
s1 = s1.toString().replace("起拍价:", "").replace("元", "").replace(",", "").replace(",", "")
|
||||
s1 = s1.toString().replace("起拍价:", "").replace("元", "").replace(",", "").replace(",", "").replace(" ","")
|
||||
if (s1.isInteger()) {
|
||||
return s1.toInteger()
|
||||
} else {
|
||||
|
||||
+348
@@ -0,0 +1,348 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import javafx.scene.input.DataFormat
|
||||
import net.sf.json.JSONObject
|
||||
import org.jsoup.Jsoup
|
||||
import org.openqa.selenium.By
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
|
||||
class IcbcCrawl {
|
||||
DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai")
|
||||
|
||||
static main(args) {
|
||||
def task = new IcbcCrawl();
|
||||
task.run();
|
||||
}
|
||||
|
||||
def run(){
|
||||
def url = "https://gf.trade.icbc.com.cn/searchproducts/pv.jhtml?resultLevel=1&FilterObjectName=%E8%B5%84%E4%BA%A7%E5%A4%84%E7%BD%AE%E5%B9%B3%E5%8F%B0&filterObjectId=8&category_Type=3&categoryId_level=&subjectChildType=1#this"
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
driver.get(url)
|
||||
driver.findElementById("category_SM310000000000000006").click()
|
||||
Thread.sleep(3000)
|
||||
|
||||
|
||||
def urls = getAllDetailLinks(driver)
|
||||
println "共:"+urls.size()
|
||||
|
||||
detailCrawling(urls, driver)
|
||||
|
||||
}
|
||||
|
||||
def getAllDetailLinks(driver){
|
||||
def stop = false
|
||||
def urls = []
|
||||
while (!stop){
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def eles = JQueryUtils.select(".tabContent5 .ymzs_sp", doc)
|
||||
if(eles){
|
||||
for(def e : eles){
|
||||
def href = e.select(".li_2_name a").attr("href")
|
||||
urls << "https://gf.trade.icbc.com.cn"+href
|
||||
}
|
||||
def nextPage = JQueryUtils.select("a:contains(下一页)", doc)
|
||||
if(nextPage){
|
||||
driver.findElement(By.linkText("下一页")).click();
|
||||
}else{
|
||||
stop = true
|
||||
}
|
||||
}else{
|
||||
stop = true
|
||||
}
|
||||
// if(urls.size()>=10){
|
||||
// stop = true
|
||||
// }
|
||||
}
|
||||
return urls;
|
||||
}
|
||||
|
||||
def detailCrawling(def urls, driver){
|
||||
def rets = []
|
||||
for(def url : urls){
|
||||
driver.get(url)
|
||||
Thread.sleep(5000)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def result = parser(doc, url)
|
||||
if (result) {
|
||||
rets << result
|
||||
println rets.size()
|
||||
}
|
||||
if (rets.size() % 100 == 0) {
|
||||
def add= saveData(rets)
|
||||
println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")+" add:"+add
|
||||
rets = []
|
||||
}
|
||||
}
|
||||
if(rets.size()>0){
|
||||
saveData(rets)
|
||||
print "完"
|
||||
}
|
||||
}
|
||||
|
||||
def parser(doc, url){
|
||||
def result = [:]
|
||||
if(doc){
|
||||
def title = JQueryUtils.selectText("#prodnameb", doc)
|
||||
def id = url.substring(url.indexOf("="), url.length())
|
||||
if(title){
|
||||
|
||||
def cityStr = JQueryUtils.selectText("dl:contains(所在城市:) li", doc)
|
||||
if(cityStr && cityStr.contains("...")){
|
||||
def cityEles = JQueryUtils.select("dl:contains(所在城市:) li", doc)
|
||||
if(cityEles){
|
||||
cityStr = cityEles[0].attr("title")
|
||||
}
|
||||
}
|
||||
if(cityStr && cityStr.contains("省")){
|
||||
result.province = cityStr.substring(0, cityStr.indexOf("省"))
|
||||
result.city = cityStr.substring(cityStr.indexOf("省")+1, cityStr.length())
|
||||
}else{
|
||||
if(cityStr.contains("自治区")){
|
||||
result.province = cityStr.substring(0, cityStr.indexOf("自治区")+3)
|
||||
result.city = cityStr.substring(cityStr.indexOf("自治区")+3, cityStr.length())
|
||||
}else{
|
||||
result.city = cityStr.substring(0, cityStr.indexOf("市"))
|
||||
}
|
||||
}
|
||||
|
||||
if(!result.city){
|
||||
return null
|
||||
}
|
||||
result.title = title
|
||||
def rMap = OpenApiUtils.matchDistrict(result.city, result.title, null, true)
|
||||
if (rMap) {
|
||||
result.matchCity = rMap.city
|
||||
result.matchRegion = rMap.region
|
||||
result.matchName = rMap.name
|
||||
result.matchAddress = rMap.address
|
||||
result.houseNum = rMap.house_num
|
||||
result.roomNum = rMap.room_num
|
||||
}
|
||||
|
||||
result.status = JQueryUtils.selectText("#message", doc).replace("本场竞买","")
|
||||
|
||||
def t1 = JQueryUtils.selectText("#pricep", doc)
|
||||
if(t1 && t1.contains("原因")){
|
||||
result.statusDesc = t1.substring(t1.indexOf("原因")+3, t1.toString().length())
|
||||
}
|
||||
result.type = JQueryUtils.selectText("dl:contains(类 型:) dd", doc)
|
||||
def enrollCount = JQueryUtils.selectText(["#signUpNum"], doc)
|
||||
if(enrollCount) {
|
||||
result.enrollCount = enrollCount.toInteger()
|
||||
}
|
||||
|
||||
def lookCount = JQueryUtils.selectText(["#visitorCount"], doc)
|
||||
if(lookCount) {
|
||||
result.lookCount = lookCount.toInteger()
|
||||
}
|
||||
|
||||
def callCount = JQueryUtils.selectText(["#remindPPNum"], doc)
|
||||
if(callCount) {
|
||||
result.callCount = callCount.toInteger()
|
||||
}
|
||||
|
||||
def curPriceStr = JQueryUtils.selectText(["#new_price_key"], doc)
|
||||
if(curPriceStr) {
|
||||
curPriceStr = curPriceStr.replace(",", "").replace("元", "").trim()
|
||||
if(curPriceStr.isInteger()) {
|
||||
result.curPrice = curPriceStr.toInteger()
|
||||
}else {
|
||||
result.curPrice = (int)(curPriceStr.toDouble())
|
||||
}
|
||||
}
|
||||
|
||||
def startPriceEles = JQueryUtils.select(["dl:contains(起始价格:) dd"], doc)
|
||||
if(startPriceEles) {
|
||||
def startPriceStr = startPriceEles[0].attr("title")
|
||||
if(startPriceStr){
|
||||
startPriceStr = startPriceStr.replace(",", "").replace("¥","")
|
||||
|
||||
if(startPriceStr.isInteger()) {
|
||||
result.startPrice = startPriceStr.toInteger()
|
||||
}else {
|
||||
result.startPrice = (int)(startPriceStr.toDouble())
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
def addPriceEles = JQueryUtils.select(["dl:contains(加价幅度:) dd"], doc)
|
||||
if(addPriceEles){
|
||||
def addPriceStr = addPriceEles[0].attr("title")
|
||||
if(addPriceStr){
|
||||
addPriceStr = addPriceStr.replace(",", "").replace("¥","")
|
||||
if(addPriceStr) {
|
||||
result.addPrice = addPriceStr.toDouble().toInteger()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def evaluationPriceEles = JQueryUtils.select(["dl:contains(评估价格:) dd"], doc)
|
||||
if(evaluationPriceEles){
|
||||
def evaluationPriceStr = evaluationPriceEles[0].attr("title")
|
||||
if(evaluationPriceStr){
|
||||
evaluationPriceStr = evaluationPriceStr.replace(",", "").replace("¥","")
|
||||
if(evaluationPriceStr.isInteger()) {
|
||||
result.evaluationPrice = evaluationPriceStr.toDouble().toInteger()
|
||||
}else {
|
||||
result.evaluationPrice = (int)(evaluationPriceStr.toDouble())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def depositPriceEles = JQueryUtils.select(["dl:contains(加价幅度:) dd"], doc)
|
||||
if(depositPriceEles){
|
||||
def v = depositPriceEles[0].attr("title")
|
||||
if(v){
|
||||
v = v.replace(",", "").replace("¥","")
|
||||
if(v.isInteger()) {
|
||||
result.depositPrice = v.toDouble().toInteger()
|
||||
}else {
|
||||
result.depositPrice = (int)(v.toDouble())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result.delayWeekly = JQueryUtils.selectText(["dl:contains(延时周期:) dd div:eq(0)"], doc)
|
||||
result.person = JQueryUtils.selectText(["dl:contains(联系人:) dd"], doc)
|
||||
result.court = JQueryUtils.selectText("#goToCourt", doc)
|
||||
def mobileEles = JQueryUtils.select("dl:contains(联系电话:) dd", doc)
|
||||
if(mobileEles){
|
||||
result.mobile = mobileEles[0].attr("title")
|
||||
}
|
||||
|
||||
def s1 = JQueryUtils.selectText("#div_announcement_gfword", doc)
|
||||
if(s1) {
|
||||
def start = s1.indexOf("法院将于")
|
||||
def end = s1.indexOf("止")
|
||||
if(start>-1 && end>-1 && end>start){
|
||||
s1 = s1.substring(s1.indexOf("法院将于"), s1.indexOf("止"))
|
||||
}
|
||||
def dateRangs = PaimaiDataUtil.getDateRange(s1)
|
||||
if(dateRangs && dateRangs.size()==2) {
|
||||
result.startDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[0], "yyyy年MM月dd日HH时")
|
||||
result.finishDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[1], "yyyy年MM月dd日HH时")
|
||||
}
|
||||
}
|
||||
|
||||
def trs = JQueryUtils.select([".rygtable table tr"], doc)
|
||||
if(trs) {
|
||||
def detailMap = [:]
|
||||
for(def tr : trs) {
|
||||
def d_tds = tr.select("td")
|
||||
if(d_tds && d_tds.size()>=2) {
|
||||
if(d_tds.size()==3) {
|
||||
if(d_tds[2].text() && d_tds[2].text()!="/"){
|
||||
detailMap.put(d_tds[1].text(), d_tds[2].text())
|
||||
}
|
||||
|
||||
}else {
|
||||
if(d_tds[1].text() && d_tds[1].text()!="/") {
|
||||
detailMap.put(d_tds[0].text(), d_tds[1].text())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if(detailMap) {
|
||||
result.detailMap = DataUtil.filterChar(detailMap)
|
||||
}
|
||||
if(!result.address) {
|
||||
result.address = detailMap.get("拍品名称")
|
||||
if(!result.address) {
|
||||
result.address = detailMap.get("标的名称")
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def imgs =JQueryUtils.select("#scroll li img", doc)
|
||||
if(imgs) {
|
||||
def imgLinks = []
|
||||
for(def img : imgs) {
|
||||
def href = img.attr("src")
|
||||
if(href.contains("icbc.com")) {
|
||||
imgLinks << "https:"+href
|
||||
}
|
||||
}
|
||||
result.imgUrls = imgLinks
|
||||
}
|
||||
|
||||
|
||||
def detailStr = JQueryUtils.selectText(".detail_list", doc)
|
||||
if(detailStr){
|
||||
def area = DataUtil.getAreaNumber(detailStr, 5000)
|
||||
if(area) {
|
||||
result.area = area.toDouble()
|
||||
if(result.tradePrice) {
|
||||
result.tradeAvgPrice = (int)(result.tradePrice/area.toDouble())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result.content = com.fangjia.core.util.DataUtil.zip(doc.toString())
|
||||
result.introduce = detailStr.toString()
|
||||
result.url = url
|
||||
result.mDate = new Date()
|
||||
result.source = "融e够"
|
||||
return result;
|
||||
}
|
||||
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def getCallCount(id){
|
||||
def url= "https://gf.trade.icbc.com.cn/wholeSaleProd/visitorCount.jhtml?prodId="+id
|
||||
def s = ABuYunUtil.doPost(url, null, null, "utf-8", true)
|
||||
if(s){
|
||||
JSONObject jsonObject = JSONObject.fromObject(s);
|
||||
println jsonObject
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def saveData(def results){
|
||||
def add = 0
|
||||
for(def ret : results){
|
||||
def q = new BasicDBObject()
|
||||
q.put("url", ret.url)
|
||||
def dbo = paimaiResultColl.findOne(q);
|
||||
|
||||
DBObject obj = new BasicDBObject();
|
||||
ret.each{k,v->
|
||||
obj.put(k,v)
|
||||
}
|
||||
|
||||
if(dbo){
|
||||
obj.m_date = new Date()
|
||||
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
|
||||
}else{
|
||||
add++
|
||||
obj.c_date = new Date()
|
||||
obj.m_date = new Date()
|
||||
try{
|
||||
paimaiResultColl.save(obj)
|
||||
}catch(Exception e){
|
||||
println obj.url
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return add
|
||||
}
|
||||
}
|
||||
+7
-2
@@ -39,8 +39,6 @@ class JdPaimaiCrawl {
|
||||
println "¹²£º"+ids.size()
|
||||
parserList( existsIds,ids, driver)
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
def getList(driver){
|
||||
@@ -478,6 +476,13 @@ class JdPaimaiCrawl {
|
||||
}
|
||||
|
||||
}
|
||||
if(result.finishDate){
|
||||
result.finishDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.finishDate, "2000-01-01")
|
||||
}
|
||||
if(result.startDate){
|
||||
result.startDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.startDate, "2050-01-01")
|
||||
}
|
||||
|
||||
result.content = com.fangjia.core.util.DataUtil.zip(doc.toString())
|
||||
result.introduce = JQueryUtils.select([".floor-content div:eq(2) table"], doc).toString()
|
||||
result.url = url
|
||||
|
||||
+7
-2
@@ -37,7 +37,7 @@ public class LianjiaCrawlByChrome {
|
||||
task.run()
|
||||
// task.run1()
|
||||
task.addDealRegion();
|
||||
// task.standardRegion()
|
||||
task.standardRegion()
|
||||
}
|
||||
|
||||
def run1(){
|
||||
@@ -227,6 +227,10 @@ public class LianjiaCrawlByChrome {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if(page>=100){
|
||||
stop = true;
|
||||
break;
|
||||
}
|
||||
page++;
|
||||
}else{
|
||||
stop = true;
|
||||
@@ -416,10 +420,11 @@ public class LianjiaCrawlByChrome {
|
||||
def cityDoc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
|
||||
def cityMap = [:]
|
||||
def citys= ["낀뉴","섈주"]
|
||||
if(cityEles){
|
||||
for(def e : cityEles) {
|
||||
def href = e.attr("href")
|
||||
if (!href.contains("fang.lianjia") && e.text().contains("¸£ÖÝ")) {
|
||||
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
|
||||
cityMap.put(e.text(), href)
|
||||
}
|
||||
}
|
||||
|
||||
+86
-1
@@ -4,9 +4,12 @@ import com.alibaba.fastjson.JSON
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import org.jsoup.Jsoup
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
@@ -19,9 +22,91 @@ class PaimaiTest {
|
||||
|
||||
DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai")
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
|
||||
|
||||
|
||||
static main(args) {
|
||||
def task = new PaimaiTest();
|
||||
task.run();
|
||||
// task.run();
|
||||
// task.stat()
|
||||
// task.updatePaimai()
|
||||
task.updateFieldPaimai()
|
||||
}
|
||||
|
||||
def updateFieldPaimai(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", new BasicDBObject('$in',["重庆","重庆市"]))
|
||||
def index = 0
|
||||
def dbcursor = paimai43_COLL.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
for(def obj : dbcursor) {
|
||||
index++
|
||||
println index
|
||||
if(obj.depositPrice){
|
||||
obj.put("depositPrice", obj.depositPrice.toInteger())
|
||||
paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def updatePaimai(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = paimaiResultColl.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def index =0
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
println index
|
||||
def query = new BasicDBObject()
|
||||
query.put("url", d.url)
|
||||
def dbo = paimai43_COLL.findOne(query);
|
||||
|
||||
DBObject obj = new BasicDBObject();
|
||||
d.each{k,v->
|
||||
obj.put(k,v)
|
||||
}
|
||||
|
||||
if(dbo){
|
||||
obj.m_date = new Date()
|
||||
paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj))
|
||||
}else{
|
||||
obj.c_date = new Date()
|
||||
obj.m_date = new Date()
|
||||
try{
|
||||
paimai43_COLL.save(obj)
|
||||
}catch(Exception e){
|
||||
println obj.url
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def stat(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", new BasicDBObject('$in', ["重庆","重庆市"]))
|
||||
def dbcursor = paimaiResultColl.find(q)
|
||||
def keys = []
|
||||
for(def db : dbcursor){
|
||||
if(db.matchCity){
|
||||
def key = db.matchCity +"##"+db.matchRegion+"##"+db.matchName;
|
||||
if(!keys.contains(key)){
|
||||
keys << key
|
||||
}
|
||||
// try{
|
||||
// def jsonObject = OpenApiClient.addressMatch("重庆", db.address, "property", false, null, null)
|
||||
// if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
// db.matchCity = jsonObject.get("city")
|
||||
// db.matchRegion = jsonObject.get("district")
|
||||
// db.matchName = jsonObject.get("name")
|
||||
// paimaiResultColl.update(new BasicDBObject("_id", db._id),new BasicDBObject('$set',db))
|
||||
// println db.address+" "+db.matchName
|
||||
// }
|
||||
// }catch(Exception e){
|
||||
// println "error "+db.address
|
||||
// }
|
||||
|
||||
}
|
||||
}
|
||||
println keys.size()
|
||||
}
|
||||
def run(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
|
||||
+416
-34
@@ -1,7 +1,12 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalUtil
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import org.jsoup.Jsoup
|
||||
import org.openqa.selenium.By
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
|
||||
class TaobaoPaimaiCrawl {
|
||||
static main(args) {
|
||||
@@ -9,55 +14,432 @@ class TaobaoPaimaiCrawl {
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def seedUrl = "https://sf.taobao.com/item_list.htm?spm=a213w.7398504.filter.86.675028eabFDx9r&category=50025969&auction_source=0&city=%C8%FD%D1%C7&sorder=2&st_param=-1&auction_start_seg=-1"
|
||||
def regionUrls = regionCrawl(seedUrl);
|
||||
def pageUrls = pageUrlCrawl(regionUrls)
|
||||
detailCrawl(pageUrls)
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
// def url = "https://sf.taobao.com/list/50025969__-1.htm?spm=a213w.7398504.filter.62.73e954b3fMWNLd&auction_source=0&st_param=-1&auction_start_seg=-1"
|
||||
def url = "https://sf.taobao.com/list/50025969__2.htm?spm=a213w.7398504.filter.65.101554b3Nynwwp&auction_source=0&st_param=-1&auction_start_seg=-1"
|
||||
WebDriver driver = new ChromeDriver()
|
||||
getAllCityRegions(driver, url);
|
||||
}
|
||||
|
||||
def regionCrawl(def seedUrl){
|
||||
def document = CrawalUtil.getDocumentByABuYun(seedUrl, "", "gbk");
|
||||
def regionUrls = []
|
||||
if(document){
|
||||
def eles = JQueryUtils.select(".small-subcondion a", document)
|
||||
if(eles){
|
||||
for(def el : eles){
|
||||
def reiongUrl = el.attr("href").trim()
|
||||
if(reiongUrl.startsWith("//")){
|
||||
regionUrls << "http:"+reiongUrl
|
||||
def getAllCityRegions(driver, url){
|
||||
|
||||
def provinces = getAllProvince(driver, url)
|
||||
def citys = ["天津","上海","北京","重庆"]
|
||||
|
||||
for(def province : provinces){
|
||||
driver.findElement(By.xpath("//a[contains(text(), '${province}')]")).click()
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles= JQueryUtils.select(".unfold .J_FilterCity a")
|
||||
for(def cityEle : cityEles){
|
||||
def city = cityEle.text()
|
||||
if(citys.contains(city)){
|
||||
driver.findElement(By.xpath("//a[contains(text(), '${province}')]")).click()
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
|
||||
driver.findElement(By.cssSelector(".J_FilterCity a")).click()
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
def regionEles= JQueryUtils.select(".J_SubCondition .small-subcondio ul li a")
|
||||
for(def r : regionEles){
|
||||
def m = [:]
|
||||
m.province = province
|
||||
m.city = city
|
||||
m.region = r.text()
|
||||
def urls = crawlLinks(m, driver)
|
||||
parse(m, driver, urls)
|
||||
}
|
||||
}else{
|
||||
def m = [:]
|
||||
m.province = province
|
||||
m.city = city
|
||||
def urls = crawlLinks(m, driver)
|
||||
parse(m, driver, urls)
|
||||
}
|
||||
}
|
||||
}
|
||||
return regionUrls;
|
||||
}
|
||||
|
||||
def pageUrlCrawl(regionUrls){
|
||||
def crawlLinks(def m, def driver){
|
||||
def stop = false
|
||||
def urls = []
|
||||
for(def url : regionUrls){
|
||||
def document = CrawalUtil.getDocumentByABuYun(url, "", "gbk");
|
||||
if(document){
|
||||
def pages = JQueryUtils.selectText(".page-total",document)
|
||||
if(pages){
|
||||
pages = pages.toInteger()
|
||||
for(def page=1;page<pages;page++){
|
||||
urls << url+"&page="+page
|
||||
while(!stop){
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def details = JQueryUtils.select(".sf-pai-item-list li a", doc)
|
||||
|
||||
for(def d : details){
|
||||
urls << "https:"+d.attr("href")
|
||||
}
|
||||
def nextEle = JQueryUtils.select(".next", doc)
|
||||
if(nextEle){
|
||||
driver.findElement(By.cssSelector(".next")).click()
|
||||
}
|
||||
}
|
||||
return urls
|
||||
}
|
||||
|
||||
def getAllProvince(driver, url){
|
||||
driver.get(url);
|
||||
Thread.sleep(100000)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def eles= JQueryUtils.select("li:contains(标的物类型) .condition li a", doc)
|
||||
def provinces = []
|
||||
for(def e : eles){
|
||||
def province = e.text()
|
||||
provinces << province
|
||||
}
|
||||
return provinces
|
||||
}
|
||||
|
||||
def parse(m, driver, urls){
|
||||
def results = []
|
||||
for(def url : urls) {
|
||||
driver.get(url);
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def result = [:]
|
||||
if (doc) {
|
||||
|
||||
def title = JQueryUtils.selectText([".pm-main h1"], doc)
|
||||
def statusDesc = JQueryUtils.selectText([".bid-fail"], doc)
|
||||
|
||||
result.province = m.province
|
||||
result.city = m.city
|
||||
if (m.region) {
|
||||
result.region = m.region
|
||||
}
|
||||
if (title) {
|
||||
|
||||
def addr = title.toString().replace("本场拍卖已", "").replace(" !", "")
|
||||
if (addr && addr.indexOf("室") > -1) {
|
||||
result.address = addr.substring(0, addr.indexOf("室") + 1)
|
||||
} else {
|
||||
result.address = addr
|
||||
}
|
||||
|
||||
if (result.city) {
|
||||
if (result.address.contains(city + "市")) {
|
||||
result.address = result.address.substring(result.address.indexOf(city) + city.length() + 1)
|
||||
} else {
|
||||
result.address = result.address.substring(result.address.indexOf(city) + city.length())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (result.address.indexOf("】") > -1) {
|
||||
result.address = result.address.substring(result.address.indexOf("】") + 1)
|
||||
}
|
||||
|
||||
|
||||
if (result.city) {
|
||||
def rMap = OpenApiUtils.matchDistrict(result.city, result.title, null, true)
|
||||
if (rMap) {
|
||||
result.matchCity = rMap.city
|
||||
result.matchRegion = rMap.region
|
||||
result.matchName = rMap.name
|
||||
result.matchAddress = rMap.address
|
||||
result.houseNum = rMap.house_num
|
||||
result.roomNum = rMap.room_num
|
||||
}
|
||||
}
|
||||
}
|
||||
if (statusDesc) {
|
||||
def status = statusDesc.substring(statusDesc.indexOf("本场拍卖") + 5, statusDesc.length() - 1)
|
||||
result.status = DataUtil.getPaiMaiType(status)
|
||||
result.statusDesc = statusDesc
|
||||
}
|
||||
|
||||
def statusEle = JQueryUtils.select(".status-tip", doc)
|
||||
if (statusEle) {
|
||||
def sEle = JQueryUtils.select(".status-ing", doc)
|
||||
if (sEle) {
|
||||
result.status = "进行中"
|
||||
} else {
|
||||
result.status = "即将开始"
|
||||
}
|
||||
}
|
||||
def curPrice = JQueryUtils.selectText(["#sf-price"], doc)
|
||||
if (curPrice) {
|
||||
if (curPrice.contains("当前价")) {
|
||||
curPrice = JQueryUtils.selectText(["#sf-price .J_Price em:eq(0)"], doc)
|
||||
if (curPrice) {
|
||||
curPrice = curPrice.replace(",", "")
|
||||
if (curPrice.isInteger()) {
|
||||
result.curPrice = curPrice.toInteger()
|
||||
} else {
|
||||
result.curPrice = (int) (curPrice.toDouble())
|
||||
}
|
||||
}
|
||||
} else if (curPrice.contains("成交价")) {
|
||||
def tradePrice = JQueryUtils.selectText(["#sf-price .J_Price em:eq(0)"], doc)
|
||||
if (tradePrice) {
|
||||
tradePrice = tradePrice.replace(",", "")
|
||||
if (tradePrice.isInteger()) {
|
||||
result.tradePrice = tradePrice.toInteger()
|
||||
} else {
|
||||
result.tradePrice = (int) (tradePrice.toDouble())
|
||||
}
|
||||
|
||||
}
|
||||
} else if (curPrice.contains("起拍价")) {
|
||||
def startPrice = JQueryUtils.selectText(["#sf-price .J_Price em:eq(0)"], doc)
|
||||
if (startPrice) {
|
||||
startPrice = startPrice.replace(",", "")
|
||||
if (startPrice.isInteger()) {
|
||||
result.startPrice = startPrice.toInteger()
|
||||
} else {
|
||||
result.startPrice = (int) (startPrice.toDouble())
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def finishStr = JQueryUtils.selectText([".pm-bid-eyebrow .over-title"], doc)
|
||||
if (finishStr) {
|
||||
if (finishStr.contains("结束时间")) {
|
||||
finishStr = JQueryUtils.select([".pm-main .J_TimeLeft"], doc).text()
|
||||
if (finishStr) {
|
||||
result.finishDate = DateUtils.parseFromTightByPattern(finishStr, "yyyy/MM/dd HH:mm:ss")
|
||||
}
|
||||
} else if (finishStr.contains("成交时间")) {
|
||||
finishStr = JQueryUtils.select([".pm-main .J_TimeLeft"], doc).text()
|
||||
if (finishStr) {
|
||||
result.finishDate = DateUtils.parseFromTightByPattern(finishStr, "yyyy/MM/dd HH:mm:ss")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def enrollCount = JQueryUtils.selectText([".pm-remind .J_Applyer"], doc)
|
||||
if (enrollCount) {
|
||||
result.enrollCount = enrollCount.toInteger()
|
||||
}
|
||||
|
||||
def callCount = JQueryUtils.selectText(["#J_NotifyNum"], doc)
|
||||
if (callCount) {
|
||||
result.callCount = callCount.toInteger()
|
||||
}
|
||||
|
||||
def lookCount = JQueryUtils.selectText(["#J_Looker"], doc)
|
||||
if (lookCount) {
|
||||
result.lookCount = lookCount.toInteger()
|
||||
}
|
||||
def tds = JQueryUtils.select(["#J_HoverShow td"], doc)
|
||||
if (tds && tds.size() > 0) {
|
||||
for (def td : tds) {
|
||||
def str = JQueryUtils.selectText("span:eq(0)", td).replace(" ", "")
|
||||
if (str) {
|
||||
if (str.contains("变卖价")) {
|
||||
def str1 = td.select(".J_Price").text()
|
||||
def str2 = td.select(".J_Price .J_Price-tip").text()
|
||||
def p = str1.replace(str2, "").replace(",", "")
|
||||
if (p) {
|
||||
if (p.isInteger()) {
|
||||
result.sellOffPrice = p.toInteger()
|
||||
} else {
|
||||
result.sellOffPrice = (int) (p.toDouble())
|
||||
}
|
||||
|
||||
}
|
||||
} else if (str.contains("加价幅度")) {
|
||||
def str1 = td.select(".J_Price").text()
|
||||
def str2 = td.select(".J_Price .J_Price-tip").text()
|
||||
def p = str1.replace(str2, "").replace(",", "")
|
||||
if (p) {
|
||||
if (p.isInteger()) {
|
||||
result.addPrice = p.toInteger()
|
||||
} else {
|
||||
result.addPrice = (int) (p.toDouble())
|
||||
}
|
||||
|
||||
}
|
||||
} else if (str.contains("类型")) {
|
||||
|
||||
result.type = td.select(".pay-type").text()
|
||||
|
||||
} else if (str.contains("保证金")) {
|
||||
def str1 = td.select(".J_Price").text()
|
||||
def str2 = td.select(".J_Price .J_Price-tip").text()
|
||||
def p = str1.replace(str2, "").replace(",", "")
|
||||
if (p) {
|
||||
if (p.isInteger()) {
|
||||
result.depositPrice = p.toInteger()
|
||||
} else {
|
||||
result.depositPrice = (int) (p.toDouble())
|
||||
}
|
||||
}
|
||||
} else if (str.contains("变卖周期")) {
|
||||
|
||||
result.sellOffWeekly = td.select("span:eq(1)").text().replace(": ", "")
|
||||
|
||||
} else if (str.contains("竞价周期")) {
|
||||
|
||||
result.pricePkWeekly = td.select("span:eq(1)").text().replace(": ", "")
|
||||
|
||||
} else if (str.contains("优先购买权")) {
|
||||
|
||||
result.preemptivePerson = td.select("span:eq(1)").text().replace(": ", "")
|
||||
|
||||
} else if (str.contains("变卖预缴款")) {
|
||||
def str1 = td.select("span:eq(1) .J_Price").text()
|
||||
def str2 = td.select("span:eq(1) .J_Price .J_Price-tip").text()
|
||||
def p = str1.replace(str2, "").replace(",", "")
|
||||
if (p) {
|
||||
if (p.isInteger()) {
|
||||
result.sellOffAdvancePay = p.toInteger()
|
||||
} else {
|
||||
result.sellOffAdvancePay = (int) (p.toDouble())
|
||||
}
|
||||
}
|
||||
|
||||
} else if (str.contains("评估价")) {
|
||||
def str1 = td.select(".J_Price").text()
|
||||
def str2 = td.select(".J_Price .J_Price-tip").text()
|
||||
def p = str1.replace(str2, "").replace(",", "")
|
||||
if (p) {
|
||||
if (p.isInteger()) {
|
||||
result.evaluationPrice = p.toInteger()
|
||||
} else {
|
||||
result.evaluationPrice = (int) (p.toDouble())
|
||||
}
|
||||
}
|
||||
} else if (str.contains("延时周期")) {
|
||||
|
||||
result.delayWeekly = td.select("span:eq(1)").text().replace(": ", "")
|
||||
|
||||
} else if (str.contains("起拍价")) {
|
||||
def str1 = td.select(".J_Price").text()
|
||||
def str2 = td.select(".J_Price .J_Price-tip").text()
|
||||
def p = str1.replace(str2, "").replace(",", "")
|
||||
if (p) {
|
||||
if (p.isInteger()) {
|
||||
result.startPrice = p.toInteger()
|
||||
} else {
|
||||
result.startPrice = (int) (p.toDouble())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result.rule = JQueryUtils.selectText(".pai-info .pai-content", doc)
|
||||
|
||||
def trs = JQueryUtils.select(["#J_desc table tr"], doc)
|
||||
if (trs) {
|
||||
def detailMap = [:]
|
||||
for (def tr : trs) {
|
||||
def d_tds = tr.select("td")
|
||||
if (d_tds && d_tds.size() >= 2) {
|
||||
if (d_tds.size() == 3) {
|
||||
detailMap.put(d_tds[1].text(), d_tds[2].text())
|
||||
} else {
|
||||
detailMap.put(d_tds[0].text(), d_tds[1].text())
|
||||
}
|
||||
} else {
|
||||
if (d_tds[0] && d_tds[0].text().contains("拍品介绍")) {
|
||||
detailMap.put("拍品介绍", d_tds[0].text().replace("拍品介绍 ", ""))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (detailMap) {
|
||||
result.detailMap = DataUtil.filterChar(detailMap)
|
||||
}
|
||||
|
||||
if (!result.address) {
|
||||
result.address = detailMap.get("拍品名称")
|
||||
if (!result.address) {
|
||||
result.address = detailMap.get("标的名称")
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def imgs = JQueryUtils.select([".slide-bigpic img"], doc)
|
||||
if (imgs) {
|
||||
def imgLinks = []
|
||||
for (def img : imgs) {
|
||||
def href = img.attr("src")
|
||||
if (href.contains("http")) {
|
||||
imgLinks << href
|
||||
} else {
|
||||
imgLinks << url.substring(0, url.indexOf(":") + 1) + href
|
||||
}
|
||||
|
||||
}
|
||||
result.imgUrls = imgLinks
|
||||
}
|
||||
}
|
||||
|
||||
//面积
|
||||
def article = JQueryUtils.selectText(["#J_ItemNotice"], doc)
|
||||
if (article) {
|
||||
def area = DataUtil.getAreaNumber(article, 5000)
|
||||
if (!area) {
|
||||
article = JQueryUtils.selectText(["#J_NoticeDetail"], doc)
|
||||
if (article) {
|
||||
area = DataUtil.getAreaNumber(article, 5000)
|
||||
}
|
||||
}
|
||||
if (!area) {
|
||||
article = JQueryUtils.selectText(["body"], doc)
|
||||
if (article) {
|
||||
area = DataUtil.getAreaNumber(article, 5000)
|
||||
}
|
||||
}
|
||||
if (area) {
|
||||
result.area = area.toDouble()
|
||||
if (result.tradePrice) {
|
||||
result.tradeAvgPrice = (int) (result.tradePrice / area.toDouble())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return urls;
|
||||
}
|
||||
|
||||
def detailCrawl(pageUrls){
|
||||
if(pageUrls){
|
||||
for(def pageUrl : pageUrls){
|
||||
def document = CrawalUtil.getDocumentByABuYun(pageUrl, "", "gbk");
|
||||
if(document){
|
||||
def pages = JQueryUtils.select(".page-total", document)
|
||||
|
||||
def s1 = JQueryUtils.selectText(["#J_NoticeDetail"], doc)
|
||||
if (s1) {
|
||||
def dateRangs = DataUtil.getDateRange(s1)
|
||||
if (dateRangs) {
|
||||
for (def j = 0; j < dateRangs.size(); j++) {
|
||||
def date = dateRangs[j]
|
||||
def pattern = null
|
||||
if (date.endsWith("日时") || date.endsWith("日")) {
|
||||
pattern = "yyyy年MM月dd日"
|
||||
} else {
|
||||
|
||||
def ind = date.indexOf("日")
|
||||
if (date.indexOf("时") == -1 && date.toString().length() > ind + 1) {
|
||||
date = date + "时"
|
||||
}
|
||||
|
||||
pattern = "yyyy年MM月dd日HH时"
|
||||
}
|
||||
if (j == 0) {
|
||||
result.startDate = DateUtils.parseFromTightByPattern(date, pattern)
|
||||
} else {
|
||||
if (!result.finishDate) {
|
||||
result.finishDate = DateUtils.parseFromTightByPattern(date, pattern)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!result.startPrice) {
|
||||
result.startPrice = DataUtil.getStartPrice(s1)
|
||||
}
|
||||
}
|
||||
|
||||
result.content = com.fangjia.core.util.DataUtil.zip(doc.toString())
|
||||
def estateType = DataUtil.findEstateType(s)
|
||||
if (estateType) {
|
||||
result.matchEstateType = estateType
|
||||
}
|
||||
|
||||
result.introduce = JQueryUtils.select(["#J_desc table"], doc).toString()
|
||||
result.url = url
|
||||
result.mDate = new Date()
|
||||
results << result
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
|
||||
|
||||
+10
-2
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.loudong
|
||||
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
@@ -16,8 +17,15 @@ class CqfdcChromeCrawl {
|
||||
|
||||
|
||||
static main(args) {
|
||||
def task = new CqfdcChromeCrawl();
|
||||
task.run()
|
||||
// def task = new CqfdcChromeCrawl();
|
||||
// task.run()
|
||||
|
||||
def name = "E3뵀짜" //69눙(방뺍C-7뵀짜) 방뺍5-5뵀짜
|
||||
def s = DataHandleUtils.findMatchPattern(name, "[\\w+]?-?\\w+[#|눙|떠|뵀]짜?.*?")
|
||||
println s
|
||||
|
||||
def unitNumStr = DataHandleUtils.findMatchPattern(name, "\\d+데禱")
|
||||
println unitNumStr
|
||||
}
|
||||
|
||||
def run(){
|
||||
|
||||
+531
-1
@@ -1,20 +1,49 @@
|
||||
package com.fangjia.script.loudong
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class CqfdcMatch {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", lineUrl)
|
||||
|
||||
static DBCollection loudongFdcConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong_cqfdc", "192.168.0.235", 27018)
|
||||
static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong", "192.168.0.235", 27018)
|
||||
|
||||
public static final DBCollection LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia_base", "loudong", lineUrl)
|
||||
public static final DBCollection HOUSE_LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", lineUrl)
|
||||
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
|
||||
static main(args) {
|
||||
def task = new CqfdcMatch();
|
||||
// task.run()
|
||||
// task.stat()
|
||||
task.export2()
|
||||
// task.export2()
|
||||
// task.matchLoudong()
|
||||
|
||||
task.addLoudong()
|
||||
// task.addFdcLoudong()
|
||||
}
|
||||
def export(){
|
||||
def q = new BasicDBObject()
|
||||
@@ -139,6 +168,378 @@ class CqfdcMatch {
|
||||
return rets;
|
||||
}
|
||||
|
||||
def addHouseLoudong(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = loudongConn.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def keys = []
|
||||
def index = 0
|
||||
for(def d : dbcursor) {
|
||||
index++
|
||||
println index
|
||||
if(d.room_num && (d.room_num.contains("商") || d.room_num.contains("社区用房"))){
|
||||
continue
|
||||
}
|
||||
if(d.area.toDouble()<=25){
|
||||
continue
|
||||
}
|
||||
def newRegion = RegionData.formatRegion(d.city, d.region)
|
||||
if (newRegion) {
|
||||
def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?")
|
||||
def house_num = null
|
||||
if (houseNumStr) {
|
||||
house_num = houseNumStr.replace("号楼", "").replace("幢", "").replace("栋", "").replace("号", "")
|
||||
}
|
||||
if (house_num) {
|
||||
def unitNum = DataHandleUtils.findMatchPattern(d.house_num, "\\d+单元")
|
||||
if (unitNum) {
|
||||
unitNum = unitNum.replace("单元", "")
|
||||
}
|
||||
def key = d.city + "##" + newRegion + "##" + d.district_name + "##" + house_num+ "##" + unitNum +"##"+d.room_num
|
||||
if(!keys.contains(key) && d.room_num) {
|
||||
keys << key
|
||||
def count = findHouseLoudongDatas(d.city, newRegion, d.district_name, house_num, unitNum, d.room_num)
|
||||
if(!count){
|
||||
def dist = null
|
||||
if(d.match_city){
|
||||
dist = CommonDatautils.findDistrict(d.match_city, d.match_region, d.match_name,"district")
|
||||
}
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("city", d.city)
|
||||
udata.put("region", newRegion)
|
||||
udata.put("name", d.district_name)
|
||||
udata.put("house_num", house_num)
|
||||
if(unitNum){
|
||||
udata.put("house_num_unit", unitNum)
|
||||
}
|
||||
udata.put("room_num", d.room_num)
|
||||
udata.put("house_area",d.area.toDouble())
|
||||
udata.put("status", 0)
|
||||
udata.put("from", "房谱网")
|
||||
if(d.floor_heigth && d.floor_heigth.isNumber()){
|
||||
udata.put("house_height", d.floor_heigth.toInteger())
|
||||
}
|
||||
if(d.floor && d.floor.isNumber()){
|
||||
udata.put("house_floor", d.floor.toInteger())
|
||||
}
|
||||
if(dist){
|
||||
udata.put("district_id", dist._id.toString())
|
||||
udata.put("name_in_seaweed", dist.name)
|
||||
}
|
||||
if(d.address){
|
||||
def address = AddressUtil.formatAddress(d.city, d.address)
|
||||
if(address){
|
||||
udata.put("address", address)
|
||||
}
|
||||
}
|
||||
udata.put("c_date", new Date())
|
||||
udata.put("m_date", new Date())
|
||||
HOUSE_LOUDONG_LINE_COLL.save(udata)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def findHouseLoudongDatas(city, region, district_name, house_num, unitNum, room_num){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", district_name)
|
||||
q.put("house_num", house_num)
|
||||
if(unitNum){
|
||||
q.put("unit_num", unitNum)
|
||||
}
|
||||
q.put("room_num", room_num)
|
||||
def count = HOUSE_LOUDONG_LINE_COLL.count(q)
|
||||
return count;
|
||||
}
|
||||
|
||||
def findAllLoundongByHouseNum(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = loudongConn.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(5000)
|
||||
def keys = []
|
||||
for(def d : dbcursor){
|
||||
def newRegion = RegionData.formatRegion(d.city, d.region)
|
||||
if(newRegion){
|
||||
def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?")
|
||||
def house_num = null
|
||||
if(houseNumStr){
|
||||
house_num = houseNumStr.replace("号楼","").replace("幢","").replace("栋","").replace("号","")
|
||||
}
|
||||
if(house_num){
|
||||
def unitNum = DataHandleUtils.findMatchPattern(d.house_num, "\\d+单元")
|
||||
if(unitNum){
|
||||
unitNum = unitNum.replace("单元","")
|
||||
}
|
||||
def key = d.city +"##"+newRegion+"##"+d.district_name+"##"+house_num
|
||||
if(unitNum){
|
||||
key = key +"##"+unitNum
|
||||
}
|
||||
if(!keys.contains(key)){
|
||||
keys << key
|
||||
def obj = findLoudongOnLine(d.city, newRegion, d.district_name, house_num, d.unit_num)
|
||||
println d.district_name+" "+d.house_num+"==${house_num}==${unitNum}"
|
||||
if(obj){
|
||||
def udata = new BasicDBObject()
|
||||
def areas = obj.areas
|
||||
if(!areas.contains(d.area)){
|
||||
areas << d.area
|
||||
|
||||
def forms = obj.from
|
||||
if(!forms.contains("房谱网")){
|
||||
forms << "房谱网"
|
||||
}
|
||||
obj.m_date = new Date()
|
||||
def query = new BasicDBObject("_id", obj._id)
|
||||
LOUDONG_LINE_COLL.update(query, new BasicDBObject('$set', obj))
|
||||
|
||||
}
|
||||
}else{
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("city", d.city)
|
||||
udata.put("region", newRegion)
|
||||
udata.put("name", d.district_name)
|
||||
udata.put("house_num", house_num)
|
||||
if(unitNum){
|
||||
udata.put("unit_num", unitNum)
|
||||
}
|
||||
if(d.address){
|
||||
def address = AddressUtil.formatAddress(d.city, d.address)
|
||||
if(address){
|
||||
udata.put("address", address)
|
||||
}
|
||||
}
|
||||
udata.put("areas",[d.area])
|
||||
udata.put("from",["房谱网"])
|
||||
if(d.floor_heigth && d.floor_heigth.toString().toInteger()){
|
||||
udata.put("floors", d.floor_heigth.toString().toInteger())
|
||||
}
|
||||
|
||||
udata.put("c_date", new Date())
|
||||
udata.put("status", 1)
|
||||
LOUDONG_LINE_COLL.save(udata)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def addFdcHouseLoudong(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = loudongFdcConn.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(2000)
|
||||
def keys = []
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
println index
|
||||
if(!d.use_type){
|
||||
continue
|
||||
}
|
||||
if(!d.use_type.contains("住宅")){
|
||||
continue
|
||||
}
|
||||
def newRegion = d.match_region
|
||||
if(newRegion){
|
||||
def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num_desc, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?")
|
||||
def house_num = null
|
||||
if(houseNumStr){
|
||||
house_num = houseNumStr.replace("号楼","").replace("幢","").replace("栋","").replace("号","")
|
||||
}
|
||||
if(house_num){
|
||||
def key = d.city +"##"+newRegion+"##"+d.name+"##"+house_num+"##"+d.unit_num+"##"+d.room_num
|
||||
if(!keys.contains(key)) {
|
||||
keys << key
|
||||
def count = findHouseLoudongDatas(d.city, newRegion, d.name, house_num, d.unit_num, d.room_num)
|
||||
if(!count){
|
||||
def maxHeight = findMaxHeight(d.city, d.name, d.house_num_desc, d.unit_num)
|
||||
def dist = null
|
||||
if(d.match_city){
|
||||
dist = CommonDatautils.findDistrict(d.match_city, d.match_region, d.match_name,"district")
|
||||
}
|
||||
def floor = null
|
||||
if(d.room_num && d.room_num.contains("-")) {
|
||||
def arrs = d.room_num.split("-")
|
||||
floor = arrs[0]
|
||||
}
|
||||
Integer room = null
|
||||
Integer hall = null
|
||||
if(d.room_type && d.room_type.contains("室")){
|
||||
def roomStr = d.room_type.substring(0,d.room_type.indexOf("室"))
|
||||
if(roomStr){
|
||||
roomStr = roomStr.replace("一","1").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("六","6").replace("七","7").replace("八","8").replace("九","9").replace("十","10")
|
||||
room = roomStr.toInteger()
|
||||
}
|
||||
|
||||
if(d.room_type.contains("厅")){
|
||||
def hallStr = d.room_type.substring(d.room_type.indexOf("室")+1, d.room_type.indexOf("厅"))
|
||||
if(hallStr){
|
||||
hallStr = hallStr.replace("一","1").replace("两","2").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("六","6").replace("七","7").replace("八","8").replace("九","9").replace("十","10")
|
||||
hall = hallStr.toInteger()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("city", d.city)
|
||||
udata.put("region", newRegion)
|
||||
udata.put("name", d.name)
|
||||
udata.put("house_num", house_num)
|
||||
if(d.unit_num){
|
||||
udata.put("house_num_unit", d.unit_num)
|
||||
}
|
||||
udata.put("room_num", d.room_num)
|
||||
udata.put("house_area",d.area.toDouble())
|
||||
udata.put("status", 0)
|
||||
udata.put("from", "重庆网上房地产")
|
||||
udata.put("house_height", maxHeight)
|
||||
if(room){
|
||||
udata.put("room", room)
|
||||
}
|
||||
if(hall){
|
||||
udata.put("hall", hall)
|
||||
}
|
||||
if(floor && floor.isNumber()){
|
||||
udata.put("house_floor", floor.toInteger())
|
||||
}
|
||||
if(dist){
|
||||
udata.put("district_id", dist._id.toString())
|
||||
udata.put("name_in_seaweed", dist.name)
|
||||
}
|
||||
if(d.address){
|
||||
def address = AddressUtil.formatAddress(d.city, d.address)
|
||||
if(address){
|
||||
udata.put("address", address)
|
||||
}
|
||||
}
|
||||
udata.put("c_date", new Date())
|
||||
udata.put("m_date", new Date())
|
||||
HOUSE_LOUDONG_LINE_COLL.save(udata)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public String getMatchString(String text,String pattern){
|
||||
Pattern p=Pattern.compile(pattern);
|
||||
Matcher m=p.matcher(text);
|
||||
if(m.find()){
|
||||
return m.group(0)
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
def addFdcBaseLoudong(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = loudongFdcConn.find(q).batchSize(5000)
|
||||
def keys = []
|
||||
for(def d : dbcursor){
|
||||
if(!d.match_region){
|
||||
continue
|
||||
}
|
||||
def newRegion = d.match_region
|
||||
|
||||
def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num_desc, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?")
|
||||
def house_num = null
|
||||
if(houseNumStr){
|
||||
house_num = houseNumStr.replace("号楼","").replace("幢","").replace("栋","").replace("号","")
|
||||
}
|
||||
if(house_num){
|
||||
def key = d.city +"##"+newRegion+"##"+d.name+"##"+house_num
|
||||
if(d.unit_num) {
|
||||
key = key + "##" + d.unit_num
|
||||
|
||||
if (!keys.contains(key)) {
|
||||
keys << key
|
||||
def obj = findLoudongOnLine(d.city, newRegion, d.name, house_num, d.unit_num)
|
||||
println d.name + " " + d.house_num_desc + "==${house_num}==${d.unit_num}"
|
||||
|
||||
def maxHeight = findMaxHeight(d.city, d.name, d.house_num_desc, d.unit_num)
|
||||
if (obj) {
|
||||
|
||||
def areas = obj.areas
|
||||
if (!areas.contains(d.area)) {
|
||||
areas << d.area
|
||||
|
||||
def forms = obj.from
|
||||
if (!forms.contains("重庆网上房地产")) {
|
||||
forms << "重庆网上房地产"
|
||||
}
|
||||
obj.m_date = new Date()
|
||||
def query = new BasicDBObject("_id", obj._id)
|
||||
LOUDONG_LINE_COLL.update(query, new BasicDBObject('$set', obj))
|
||||
|
||||
}
|
||||
} else {
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("city", d.city)
|
||||
udata.put("region", newRegion)
|
||||
udata.put("name", d.name)
|
||||
udata.put("house_num", house_num)
|
||||
if (d.unit_num) {
|
||||
udata.put("unit_num", d.unit_num)
|
||||
}
|
||||
if (d.address) {
|
||||
def address = AddressUtil.formatAddress(d.city, d.address)
|
||||
if (address) {
|
||||
udata.put("address", address)
|
||||
}
|
||||
}
|
||||
udata.put("areas", [d.area])
|
||||
udata.put("from", ["重庆网上房地产"])
|
||||
udata.put("floors", maxHeight)
|
||||
udata.put("c_date", new Date())
|
||||
udata.put("status", 1)
|
||||
LOUDONG_LINE_COLL.save(udata)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def findMaxHeight(city, name,house_num, unit_num){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",city)
|
||||
q.put("name",name)
|
||||
q.put("house_num_desc",house_num)
|
||||
q.put("unit_num",unit_num)
|
||||
def dbcursor = loudongFdcConn.find(q)
|
||||
def heights = []
|
||||
for(def h : dbcursor){
|
||||
if(h.nominal_floor && h.nominal_floor.isNumber()){
|
||||
def height = h.nominal_floor.toInteger()
|
||||
if(!heights.contains(height)){
|
||||
heights << height
|
||||
}
|
||||
}
|
||||
}
|
||||
if(heights){
|
||||
heights = heights.sort()
|
||||
return heights[heights.size()-1]
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def findLoudongOnLine(city, region, name, house_num, unit_num){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",city)
|
||||
q.put("region",region)
|
||||
q.put("name",name)
|
||||
q.put("house_num",house_num)
|
||||
if(unit_num){
|
||||
q.put("unit_num",unit_num)
|
||||
}
|
||||
def dbo = LOUDONG_LINE_COLL.findOne(q)
|
||||
return dbo
|
||||
}
|
||||
|
||||
|
||||
|
||||
def stat(){
|
||||
def q = new BasicDBObject()
|
||||
@@ -168,6 +569,135 @@ class CqfdcMatch {
|
||||
|
||||
}
|
||||
|
||||
def matchLoudong(){
|
||||
def names = findAllLoudong("重庆")
|
||||
print names.size()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(names.size())
|
||||
|
||||
def retMap = new ConcurrentHashMap()
|
||||
for(def i=0;i<names.size();i++){
|
||||
def key = names[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def arrs = key.split("#")
|
||||
def jsonObject = OpenApiClient.addressMatch(arrs[0], arrs[2], "property", false, arrs[1], null)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
|
||||
def m = [:]
|
||||
m.id = jsonObject.get("id")
|
||||
m.city = jsonObject.get("city")
|
||||
m.region = jsonObject.get("district")
|
||||
m.name = jsonObject.get("name")
|
||||
def dist = CommonDatautils.findDistrictById(m.id)
|
||||
if(dist){
|
||||
m.sid = dist.sid
|
||||
}
|
||||
retMap.put(key, m)
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
println "匹配后:"+retMap.size()
|
||||
updateLoudong(retMap)
|
||||
println "修改完成!"
|
||||
}
|
||||
|
||||
def updateLoudong(retMap){
|
||||
def index = 0
|
||||
retMap.each{key,m->
|
||||
def arrs = key.split("#")
|
||||
|
||||
def q = new BasicDBObject("city", arrs[0])
|
||||
q.put('region', arrs[1]);
|
||||
q.put('name', arrs[2]);
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("district_id", m.id.toString())
|
||||
udata.put("name_in_seaweed", m.name)
|
||||
udata.put("district_no", m.sid)
|
||||
HOUSE_LOUDONG_COLL.update(q, new BasicDBObject('$set', udata), false, true);
|
||||
index++
|
||||
println index
|
||||
}
|
||||
}
|
||||
|
||||
def findAllLoudong(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("district_id",new BasicDBObject('$exists', false))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city", 1)
|
||||
keys.put("region", 1)
|
||||
keys.put("name", 1)
|
||||
def dbcursor = HOUSE_LOUDONG_COLL.find(q).batchSize(5000)
|
||||
def names = new CopyOnWriteArrayList()
|
||||
for(def db : dbcursor){
|
||||
def key = db.city +"#"+db.region+"#"+db.name
|
||||
if(!names.contains(key)){
|
||||
names << key
|
||||
}
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
def addLoudong(){
|
||||
// def stat = statBaseLoudong()
|
||||
// println stat
|
||||
// findAllLoundongByHouseNum();
|
||||
def stat1 = statHouseLoudong()
|
||||
println stat1
|
||||
// addHouseLoudong()
|
||||
}
|
||||
|
||||
def addFdcLoudong(){
|
||||
// addFdcBaseLoudong()
|
||||
addFdcHouseLoudong()
|
||||
}
|
||||
|
||||
def statHouseLoudong(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", "重庆")
|
||||
def dbcursor = HOUSE_LOUDONG_LINE_COLL.find(q)
|
||||
def total = dbcursor.size()
|
||||
def ids = []
|
||||
for(def d : dbcursor){
|
||||
if(d.district_id && !ids.contains(d.district_id)){
|
||||
ids << d.district_id
|
||||
}
|
||||
}
|
||||
def map = [:]
|
||||
map.rows_total = total
|
||||
map.dist_count = ids.size()
|
||||
return map
|
||||
}
|
||||
|
||||
def statBaseLoudong(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", "重庆")
|
||||
def dbcursor = LOUDONG_LINE_COLL.find(q)
|
||||
def total = dbcursor.size()
|
||||
def ids = []
|
||||
for(def d : dbcursor){
|
||||
if(d.id_in_seaweed && !ids.contains(d.id_in_seaweed)){
|
||||
ids << d.id_in_seaweed
|
||||
}
|
||||
}
|
||||
def map = [:]
|
||||
map.rows_total = total
|
||||
map.dist_count = ids.size()
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
+2
-1
@@ -12,7 +12,8 @@ import com.fangjia.script.util.DateUtils
|
||||
class DingTalkUtils {
|
||||
|
||||
static main(String[] args) {
|
||||
Date now = DateUtils.parseFromTightByPattern("2021-08-01", "yyyy-MM-dd") ;
|
||||
// 要导出的月份第一天
|
||||
Date now = DateUtils.parseFromTightByPattern("2022-03-01", "yyyy-MM-dd") ;
|
||||
|
||||
String startTime = DateUtils.addMonth(now, -1) + " 00:00:00"
|
||||
String monthStartTime = DateUtils.formatDateByPattern(now, "yyyy-MM-dd HH:mm:ss")
|
||||
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
package com.fangjia.script.temp.shenjian.geo
|
||||
|
||||
import com.fangjia.script.gj.location.Point
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.xh.InsidePolygon
|
||||
import com.mongodb.BasicDBList
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "block", "192.168.0.85", 28017)
|
||||
|
||||
collection.find(new BasicDBObject()).forEach({ obj ->
|
||||
if (obj.type == 'block') {
|
||||
if (obj.region instanceof String) {
|
||||
return;
|
||||
}
|
||||
BasicDBList reigon = (BasicDBList)obj.region
|
||||
int count = 0
|
||||
List<String> info = new ArrayList<>()
|
||||
for (String s : reigon) {
|
||||
info.add(s)
|
||||
}
|
||||
if (info.size() > 1) {
|
||||
boolean find = false
|
||||
for (String s : info) {
|
||||
if (find) {
|
||||
break
|
||||
}
|
||||
collection.find(new BasicDBObject("type", "region").append("name", s)).limit(1).forEach({obj2 ->
|
||||
String border = obj2.border
|
||||
String points = border.replaceAll(",", "-").replaceAll(";", "-")
|
||||
List<com.fangjia.script.looplineinfo.Point> pointList = InsidePolygon.createPoint(points);
|
||||
com.fangjia.script.looplineinfo.Point point = new com.fangjia.script.looplineinfo.Point(obj.lng, obj.lat)
|
||||
if (InsidePolygon.aboutZone(point, pointList)) {
|
||||
collection.update(new BasicDBObject("_id", obj._id),
|
||||
new BasicDBObject('$set', new BasicDBObject("region", obj2.name)));
|
||||
find = true;
|
||||
}
|
||||
})
|
||||
}
|
||||
} else {
|
||||
collection.update(new BasicDBObject("_id", obj._id),
|
||||
new BasicDBObject('$set', new BasicDBObject("region", info.get(0))));
|
||||
}
|
||||
}
|
||||
})
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
package com.fangjia.script.temp.shenjian.poi
|
||||
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.statistics.poi.AmapPoi
|
||||
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.apache.commons.collections.CollectionUtils
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
final DBCollection ZS_REGION_CONN = MongoUtils.getCollectionByName2("fangjia", "region_block", ZS_MONGODB_URL)
|
||||
|
||||
ZS_REGION_CONN.find(new BasicDBObject()).batchSize(1000).each {obj ->
|
||||
def city = obj.city
|
||||
def org_name = obj.org_name
|
||||
|
||||
List<AmapPoi> result = GaoDeUtils.searchPoi(city, org_name, org_name, null)
|
||||
if (CollectionUtils.isNotEmpty(result)) {
|
||||
def location = result.get(0).location
|
||||
def lon = location.split(",")[0]
|
||||
def lat = location.split(",")[1]
|
||||
ZS_REGION_CONN.update(new BasicDBObject("_id", obj._id),
|
||||
new BasicDBObject('$set',new BasicDBObject("location", new BasicDBObject("lon", lon).append("lat", lat))))
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -128,10 +128,10 @@ class SeaweedUtils {
|
||||
*/
|
||||
static formatHouseholdCount(def household_count) {
|
||||
if (household_count && FieldValidatorUtils.checkValue(household_count) ) {
|
||||
if (Long.valueOf(household_count) < 50) {
|
||||
if (Integer.valueOf(household_count) < 50) {
|
||||
return "" // »§Êý
|
||||
}
|
||||
return household_count
|
||||
return household_count.toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+28
-28
@@ -40,13 +40,13 @@ class Esf58Crawl extends BaseBatch{
|
||||
def static useDistKeys = []
|
||||
static def totalCount = new AtomicLong()
|
||||
|
||||
def cityList = ["成都":"cd|102","杭州":"hz|79","重庆":"cq|37","苏州":"su|5","无锡":"wx|93", "深圳":"sz|4","郑州":"zz|342", "长沙":"cs|414","东莞":"dg|413",
|
||||
def cityList = ["福州":"fz|304","成都":"cd|102","杭州":"hz|79","重庆":"cq|37","苏州":"su|5","无锡":"wx|93", "深圳":"sz|4","郑州":"zz|342", "长沙":"cs|414","东莞":"dg|413",
|
||||
"괏땍":"bd|424","괏癸":"baoting|10367","켓퀼":"nn|845", "爛鹵":"yz|637","�庫":"st|783","揭鹵":"taizhou|693","뺨갛":"ha|968","纜쉭":"zhanjiang|791","쏩鹵":"jinzhou|2354","웹츠":"km|541","진鹵":"liuzhou|7133","凉헤":"zq|901","젯暾멍":"lyg|2049","휑亮":"rizhao|3177","끽돠":"changde|872",
|
||||
"꾕鹵":"cangzhou|652", "홋鹵":"qz|291","헌陶":"qingyuan|7303", "鮫퀼":"xn|2052","潾屢":"zunyi|7620","瓏碌":"zhuzhou|1086","뱍捺":"gy|2015","纛捺":"yy|821","其鹵":"xz|471", "벗랑깝":"hrb|202","露끗":"yc|858", "읜렌":"lf|772","해뽈돎":"qhd|1078","쁨鹵":"huizhou|722","쭤捺":"luoyang|556",
|
||||
"柯소鏤":"sjz|241","格覩":"ty|740","吉냘":"yancheng|613","斤捺":"xy|8694","켓념":"nanchong|2378","켓捺":"ny|592","衢쳔":"xm|606",
|
||||
"축捺":"mianyang|1057","셌퀼":"jining|450","벳덱":"hd|572",
|
||||
"뺨굇":"huaibei|9357","뇔鹵":"chuzhou|10266",
|
||||
"福州":"fz|304","蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158",
|
||||
"蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158",
|
||||
"읊鹵":"lz|952","북례":"hf|837", "魁쨀컁펭":"xj|984","켓끗":"nc|669","�捺":"sy|188","鮫갛":"xa|483","藿렌":"wf|362",
|
||||
"交빤":"wuhu|2045","땜賻":"dg|413","끽鹵":"cz|463","뤘�":"fs|222",
|
||||
"켓繫":"nt|394","섈주":"jl|700","櫓�":"zs|771","막捺":"fy|2325",
|
||||
@@ -192,37 +192,37 @@ class Esf58Crawl extends BaseBatch{
|
||||
void process(Object args) {
|
||||
//addContent("르鹵","https://fz.58.com/xiaoqu/", 70);
|
||||
|
||||
def citys = ["佛山","潍坊","长春","太原","南通","厦门","呼和浩特","常州","中山","德阳","镇江","泰安","银川","襄阳","江门","荆州","上海","衡阳","滨州","遂宁","鞍山","赣州","阜阳","枣庄","聊城","芜湖","吉林","德州","信阳","新乡","宿迁","沧州","桂林","包头","常德","乐山","遵义","眉山","株洲","驻马店","日照","许昌","成都","衡水","宜宾","泸州","营口","达州","宝鸡","广安","吉安","孝感","鄂尔多斯","十堰","清远","湘潭","上饶","漯河","邢台","平顶山","菏泽","宜春","承德","赤峰","延边","茂名","焦作","内江","韶关","梅州","郴州","黄冈","红河","绥化","娄底","丹东","邵阳","开封","阳江","黄石","周口","牡丹江","长治","咸宁","汉中","自贡","衢州","晋中","永州","北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏","重庆"]
|
||||
for(def city : citys){
|
||||
def str = cityList.get(city)
|
||||
for(def i=0;i<1;i++){
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
|
||||
if(districts){
|
||||
def arrs = str.split("\\|")
|
||||
def url = "https://"+arrs[0]+".58.com/xiaoqu/"
|
||||
distMap.put(city, districts)
|
||||
addCrawlKeys(city);
|
||||
addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// cityList.each{city, str->
|
||||
// def citys = ["北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏"]
|
||||
// for(def city : citys){
|
||||
// def str = cityList.get(city)
|
||||
// for(def i=0;i<1;i++){
|
||||
// if(citys.contains(city)){
|
||||
// def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
|
||||
// if(districts){
|
||||
// def arrs = str.split("\\|")
|
||||
// def url = "https://"+arrs[0]+".58.com/xiaoqu/"
|
||||
// distMap.put(city, districts)
|
||||
//
|
||||
// addCrawlKeys(city);
|
||||
// addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
|
||||
// }
|
||||
// def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
|
||||
// if(districts){
|
||||
// def arrs = str.split("\\|")
|
||||
// def url = "https://"+arrs[0]+".58.com/xiaoqu/"
|
||||
// distMap.put(city, districts)
|
||||
// addCrawlKeys(city);
|
||||
// addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
cityList.each{city, str->
|
||||
for(def i=0;i<1;i++){
|
||||
// if(citys.contains(city)){
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
|
||||
if(districts){
|
||||
def arrs = str.split("\\|")
|
||||
def url = "https://"+arrs[0]+".58.com/xiaoqu/"
|
||||
distMap.put(city, districts)
|
||||
|
||||
addCrawlKeys(city);
|
||||
addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
+1
-1
@@ -18,7 +18,7 @@ class SynchDistrictPrice {
|
||||
// def sourceMap = ["fang.com":"搜房","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
|
||||
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"]
|
||||
//"fang.com":"搜房",
|
||||
def sourceMap = ["ke.com":"贝壳找房","fang.com":"搜房","cityhouse.cn":"城市房产"]
|
||||
def sourceMap = ["fang.com":"搜房","ke.com":"贝壳找房","5i5j.com":"我爱我家"]
|
||||
static main(args) {
|
||||
def task = new SynchDistrictPrice()
|
||||
task.run()
|
||||
|
||||
+2
-2
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
|
||||
def run(){
|
||||
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
|
||||
for(def source :sources){
|
||||
def count1 = findTotalCount(source, 202202);
|
||||
def count2 = findTotalCount(source, 202203);
|
||||
def count1 = findTotalCount(source, 202203);
|
||||
def count2 = findTotalCount(source, 202204);
|
||||
println source +" "+count1 +" "+count2
|
||||
}
|
||||
}
|
||||
|
||||
@@ -245,6 +245,48 @@ public class CommonUtils {
|
||||
|
||||
}
|
||||
|
||||
|
||||
static void recalculationSqRent(String city, String region, String name, String estateType, Date calcDate = new Date(), boolean isTest = false, int calcType = 0, needRedrawTrend = true, def sync = false) {
|
||||
|
||||
|
||||
calcDate = com.fangjia.core.util.DateUtil.truncateDay(calcDate)
|
||||
if (estateType && estateType.contains("±ðÊû")) {
|
||||
estateType = "±ðÊû"
|
||||
} else if (estateType) {
|
||||
estateType = "Ð¡Çø"
|
||||
}
|
||||
|
||||
DBObject cmd = new BasicDBObject()
|
||||
cmd.put("city", city)
|
||||
cmd.put("region", region)
|
||||
cmd.put("name", name)
|
||||
cmd.put('estate_type', estateType)
|
||||
cmd.put("calc_date", calcDate)
|
||||
cmd.put("calc_type", 0)
|
||||
println(cmd)
|
||||
DBObject data = new BasicDBObject()
|
||||
data.put("category", "square_rent")
|
||||
data.put("priority", 3)
|
||||
data.put("status", 0)
|
||||
data.put("remark", 'week')
|
||||
data.put("c_date", new Date())
|
||||
|
||||
|
||||
if(isTest){
|
||||
data.put("ip", "192.168.0.231")
|
||||
COMMAND_COLL.update(cmd, new BasicDBObject('$set', data), true, false)
|
||||
}else{
|
||||
data.put("ip", ["fangjia-priceanalysis-0", "fangjia-priceanalysis-1"][new Random().nextInt(2)])
|
||||
COMMAND_COLL.update(cmd, new BasicDBObject('$set', data), true, false)
|
||||
}
|
||||
if(sync){
|
||||
while (COMMAND_COLL.count(cmd) != 0){
|
||||
Thread.sleep(1000)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* 重算小区均价
|
||||
* @param city
|
||||
|
||||
@@ -305,6 +305,43 @@ public class DbUtils{
|
||||
}
|
||||
return districts
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询附近一共里内小区的数量
|
||||
*/
|
||||
def static findAroundDistrictByRadius(BasicDBObject district, def radius){
|
||||
def districts = []
|
||||
if(hasGeo(district)){
|
||||
def query = new BasicDBObject()
|
||||
query.append("cat", "district")
|
||||
query.append("city", district.city)
|
||||
query.append("visible", 0)
|
||||
DBObject sdq = new BasicDBObject();
|
||||
def lat = district.lat / 10000000000.0
|
||||
def lng = district.lng / 10000000000.0
|
||||
sdq.put("sd.point", lat + "," + lng);
|
||||
sdq.put("sd.sfield", "lat_lng");
|
||||
sdq.put("sd.distance", String.valueOf((double) radius / 1000.0));
|
||||
query.append('$sd', sdq)
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.append("city", 1)
|
||||
keys.append("region", 1)
|
||||
keys.append("name", 1)
|
||||
keys.append("lat", 1)
|
||||
keys.append("lng", 1)
|
||||
keys.append("lat2", 1)
|
||||
keys.append("lng2", 1)
|
||||
keys.append("estate_type2", 1)
|
||||
DbUtils.SEAWEED_SLAVE.find(query, keys).each {BasicDBObject dbo->
|
||||
if(!district._id || dbo._id != district._id){
|
||||
districts << dbo
|
||||
}
|
||||
}
|
||||
}
|
||||
return districts
|
||||
|
||||
}
|
||||
/**
|
||||
* 查询小区均价
|
||||
|
||||
@@ -229,6 +229,93 @@ public class OpenApiClient {
|
||||
return null
|
||||
}
|
||||
|
||||
public static JSONObject searchLoudong(String id, String houseNum, String unitNum){
|
||||
try{
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("id", id);
|
||||
params.put("houseNumber", houseNum);
|
||||
params.put("unitNumber", unitNum);
|
||||
params.put("token", TOKEN);
|
||||
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/property/searchLoudong", params, "utf-8")
|
||||
JSONObject result = JSONObject.fromObject(content);
|
||||
while (result == null) {
|
||||
result = HttpRequestUtil.getJsonContent("http://localhost:8080/fangjia-open/property/searchLoudong", params, "utf-8")
|
||||
println("${id} ${houseNum} 程序调用异常")
|
||||
Thread.sleep(6000)
|
||||
}
|
||||
if(result){
|
||||
if (result.getInt("code") == 200) {
|
||||
return result;
|
||||
} else if (result.getInt("code") !=0) {
|
||||
println("${id} ${houseNum}")
|
||||
}
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
println("${id} ${houseNum} ${e.getMessage()}")
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
public static JSONObject searchHouseLoudong(String id, String houseNum, String unitNum, String roomNum){
|
||||
try{
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("id", id);
|
||||
params.put("houseNumber", houseNum);
|
||||
params.put("unitNumber", unitNum);
|
||||
params.put("roomNumber", roomNum);
|
||||
params.put("token", TOKEN);
|
||||
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/property/searchHouseLoudong", params, "utf-8")
|
||||
JSONObject result = JSONObject.fromObject(content);
|
||||
while (result == null) {
|
||||
result = HttpRequestUtil.getJsonContent("http://localhost:8080/fangjia-open/property/searchHouseLoudong", params, "utf-8")
|
||||
println("${id} ${houseNum} 程序调用异常")
|
||||
Thread.sleep(6000)
|
||||
}
|
||||
if(result){
|
||||
if (result.getInt("code") == 200) {
|
||||
return result;
|
||||
} else if (result.getInt("code") !=0) {
|
||||
println("${id} ${houseNum}")
|
||||
}
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
println("${id} ${houseNum} ${e.getMessage()}")
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
public static JSONObject standard2Action(String city, String address) {
|
||||
try {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("city", city);
|
||||
params.put("address", address);
|
||||
params.put("token", TOKEN);
|
||||
params.put("__refresh_cache","1")
|
||||
|
||||
String content = new HttpRequest().get("https://open.fangjia.com/v2/address/standard", params, "utf-8")
|
||||
JSONObject result = JSONObject.fromObject(content);
|
||||
while (result == null) {
|
||||
result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/v2/address/standard", params, "utf-8")
|
||||
println("${city} ${address} 程序调用异常")
|
||||
Thread.sleep(6000)
|
||||
}
|
||||
if (result.getInt("code") == 0) {
|
||||
return result.getJSONObject("result");
|
||||
} else if (result.getInt("code") !=0) {
|
||||
println("${city} ${address}")
|
||||
}
|
||||
} catch (Exception e) {
|
||||
println("${city} ${address} ${e.getMessage()}")
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
public static JSONObject standardize(String city, String address) {
|
||||
try {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
@@ -593,6 +680,90 @@ public class OpenApiClient {
|
||||
}
|
||||
|
||||
|
||||
static JSONObject facilitys(String city, String region, String name) {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("token", TOKEN);
|
||||
params.put("city", city)
|
||||
params.put("region", region)
|
||||
params.put("name", name)
|
||||
try {
|
||||
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/district/facilitys", params, "utf-8")
|
||||
JSONObject result = JSONObject.fromObject(content);
|
||||
|
||||
return result
|
||||
} catch (Exception e) {
|
||||
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static JSONObject searchPaimais(String city, String region, String name, limit) {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("token", TOKEN);
|
||||
params.put("city", city)
|
||||
params.put("district", region)
|
||||
params.put("name", name)
|
||||
params.put("limit", "10")
|
||||
try {
|
||||
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/auction/searchPaimais", params, "utf-8")
|
||||
JSONObject result = JSONObject.fromObject(content);
|
||||
return result
|
||||
} catch (Exception e) {
|
||||
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
static JSONObject searchDealPrice(String id, String startDate, limit) {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("token", TOKEN);
|
||||
params.put("id", id)
|
||||
params.put("startDate", startDate)
|
||||
params.put("limit", limit.toString())
|
||||
try {
|
||||
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/dealPrice/searchList", params, "utf-8")
|
||||
JSONObject result = JSONObject.fromObject(content);
|
||||
return result
|
||||
} catch (Exception e) {
|
||||
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static JSONObject houseList(String id, String startDate, limit) {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("token", TOKEN);
|
||||
params.put("id", id)
|
||||
params.put("startTime", startDate)
|
||||
params.put("pageSize", limit.toString())
|
||||
try {
|
||||
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/house/list", params, "utf-8")
|
||||
JSONObject result = JSONObject.fromObject(content);
|
||||
return result
|
||||
} catch (Exception e) {
|
||||
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
static JSONObject evaluaionList(String id, String startDate, limit) {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("token", TOKEN);
|
||||
params.put("id", id)
|
||||
params.put("startDate", startDate)
|
||||
params.put("limit", limit.toString())
|
||||
try {
|
||||
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/evaluation/bsList", params, "utf-8")
|
||||
JSONObject result = JSONObject.fromObject(content);
|
||||
return result
|
||||
} catch (Exception e) {
|
||||
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static JSONObject districtDetail(String city, String region, String name) {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("token", TOKEN);
|
||||
|
||||
+2
-2
@@ -11,8 +11,8 @@ import org.bson.Document
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
def startDate = DateUtil.parseDate("20220301", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20220401", "yyyyMMdd")
|
||||
def startDate = DateUtil.parseDate("20220401", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20220501", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
package com.fangjia.service.datastore;
|
||||
|
||||
import com.fangjia.dal.mongo.model.BsEvaluation;
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
public interface BsEvaluationService {
|
||||
|
||||
List<BsEvaluation> getNearlySameEvaluation(String id, Integer minPrice, Integer maxPrice, Date startDate, Date endDate, Integer limit);
|
||||
}
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
package com.fangjia.service.datastore;
|
||||
|
||||
import com.fangjia.dal.QueryParam;
|
||||
import com.fangjia.dal.SortParam;
|
||||
import com.fangjia.dal.mongo.dao.BsEvaluationDao;
|
||||
import com.fangjia.dal.mongo.dao.EvaluationDao;
|
||||
import com.fangjia.dal.mongo.model.BsEvaluation;
|
||||
import com.fangjia.dal.mongo.model.bkoff.DealPrice;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
public class BsEvaluationServiceImpl implements BsEvaluationService {
|
||||
|
||||
private BsEvaluationDao bsEvaluationDao;
|
||||
|
||||
public void setBsEvaluationDao(BsEvaluationDao bsEvaluationDao) {
|
||||
this.bsEvaluationDao = bsEvaluationDao;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<BsEvaluation> getNearlySameEvaluation(String id, Integer minPrice, Integer maxPrice, Date startDate, Date endDate, Integer limit) {
|
||||
List<QueryParam> params = new ArrayList<QueryParam>();
|
||||
params.add(new QueryParam("commid", id));
|
||||
if(startDate!=null){
|
||||
params.add(new QueryParam("cDate", QueryParam.GREATER_THAN_EQUAL_OPERATOR, startDate));
|
||||
}
|
||||
if(endDate!=null){
|
||||
params.add(new QueryParam("cDate", QueryParam.LESS_THAN_EQUAL_OPERATOR, startDate));
|
||||
}
|
||||
|
||||
if (minPrice > 0) {
|
||||
params.add(new QueryParam("unitPrice", QueryParam.GREATER_THAN_OPERATOR, minPrice));
|
||||
}
|
||||
if (maxPrice > 0) {
|
||||
params.add(new QueryParam("unitPrice", QueryParam.LESS_THAN_OPERATOR, maxPrice));
|
||||
}
|
||||
List<SortParam> sortParams = new ArrayList<SortParam>();
|
||||
sortParams.add(new SortParam("cDate", false));
|
||||
List<BsEvaluation> bsEvaluations = bsEvaluationDao.findAllByParams(params, sortParams, limit);
|
||||
return bsEvaluations;
|
||||
}
|
||||
}
|
||||
+2
@@ -94,4 +94,6 @@ public interface HouseLoudongService {
|
||||
public List<HouseLoudong> findAll(List<QueryParam> queryParams, Pager pager);
|
||||
|
||||
List<String> findAllSources();
|
||||
|
||||
public List<HouseLoudong> findAll(String id, String houseNum, String unitNum, String roomNum);
|
||||
}
|
||||
|
||||
+16
@@ -297,6 +297,22 @@ public class HouseLoudongServiceImpl implements HouseLoudongService {
|
||||
return froms;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<HouseLoudong> findAll(String id, String houseNum, String unitNum, String roomNum) {
|
||||
List<QueryParam> params = new ArrayList<QueryParam>();
|
||||
params.add(new QueryParam("districtId", id));
|
||||
if (StringUtils.isNotBlank(houseNum)) {
|
||||
params.add(new QueryParam("houseNum", houseNum));
|
||||
}
|
||||
if (StringUtils.isNotBlank(unitNum)) {
|
||||
params.add(new QueryParam("houseNumUnit", unitNum));
|
||||
}
|
||||
if (StringUtils.isNotBlank(roomNum)) {
|
||||
params.add(new QueryParam("roomNum", roomNum));
|
||||
}
|
||||
return houseLoudongBkoffDao.findAllByParams(params);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<HouseLoudong> queryHouseLoudongList(String city, String districtName, String roomNum, Pager pager) {
|
||||
List<QueryParam> params = new ArrayList<QueryParam>();
|
||||
|
||||
+4
@@ -23,6 +23,8 @@ public interface LoudongService {
|
||||
|
||||
List<Loudong> findAll(String city, String region, String name, String houseNumber, String unitNumber, int limit);
|
||||
|
||||
List<Loudong> findAll(String id, String houseNumber, String unitNumber);
|
||||
|
||||
Map<String, XSSFWorkbook> findAll(List<QueryParam> query, String city);
|
||||
|
||||
Loudong findById(String id);
|
||||
@@ -41,4 +43,6 @@ public interface LoudongService {
|
||||
void save(Map<String, Object> data);
|
||||
|
||||
List<String> findAllSources();
|
||||
|
||||
Loudong findForEvaluate(String idInSeaweed, String houseNumber, String unitNumber);
|
||||
}
|
||||
|
||||
+36
@@ -148,6 +148,23 @@ public class LoudongServiceImpl implements LoudongService {
|
||||
return loudongDao.findAllByParams(params,null, 0 , limit, ADDRESS_VERIFY_FIELDS);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Loudong> findAll(String id, String houseNumber, String unitNumber) {
|
||||
List<QueryParam> params = new ArrayList<QueryParam>();
|
||||
if(StringUtils.isNotBlank(id)) {
|
||||
params.add(new QueryParam("idInSeaweed", id));
|
||||
}
|
||||
|
||||
if(StringUtils.isNotBlank(houseNumber)) {
|
||||
params.add(new QueryParam("houseNumber", houseNumber));
|
||||
}
|
||||
if(StringUtils.isNotBlank(unitNumber)) {
|
||||
params.add(new QueryParam("unitNumber", unitNumber));
|
||||
}
|
||||
|
||||
return loudongDao.findAllByParams(params);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Loudong findById(String id) {
|
||||
return loudongDao.findById(id);
|
||||
@@ -318,4 +335,23 @@ public class LoudongServiceImpl implements LoudongService {
|
||||
}
|
||||
return froms;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Loudong findForEvaluate(String idInSeaweed, String houseNumber, String unitNumber){
|
||||
if(StringUtils.isBlank(idInSeaweed) || StringUtils.isBlank(houseNumber)){
|
||||
return null;
|
||||
}
|
||||
List<QueryParam> params = Lists.newArrayList();
|
||||
params.add(new QueryParam("idInSeaweed", idInSeaweed));
|
||||
params.add(new QueryParam("houseNumber", houseNumber));
|
||||
if(StringUtils.isNotBlank(unitNumber)) {
|
||||
params.add(new QueryParam("unitNumber", unitNumber));
|
||||
}
|
||||
params.add(new QueryParam("floors", QueryParam.GREATER_THAN_OPERATOR, 0));
|
||||
params.add(new QueryParam("floors", QueryParam.LESS_THAN_OPERATOR, 60));
|
||||
params.add(new QueryParam("status", QueryParam.NOT_EQUAL_OPERATOR, 0));
|
||||
|
||||
List<SortParam> sortParams = ImmutableList.of(new SortParam("floors", false));
|
||||
return loudongDao.findFirstByParams(params, sortParams);
|
||||
}
|
||||
}
|
||||
|
||||
+2
@@ -82,6 +82,8 @@ public interface DealPriceService {
|
||||
|
||||
public List<DealPrice> getNearlySameDealPrice(DistrictKey key, int minPrice, int maxPrice, int radius, int limit);
|
||||
|
||||
public List<DealPrice> getNearlySameDealPrice(String districtId, Integer minPrice, Integer maxPrice,Date startDate, Date endDate, int limit);
|
||||
|
||||
public List<DealPrice> getDisSameDealPrice(DistrictKey key, int minPrice, int maxPrice, int limit);
|
||||
|
||||
/**
|
||||
|
||||
+23
@@ -354,6 +354,29 @@ public class DealPriceServiceImpl implements DealPriceService {
|
||||
return houses;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DealPrice> getNearlySameDealPrice(String districtId, Integer minPrice, Integer maxPrice, Date startDate, Date endDate, int limit) {
|
||||
List<QueryParam> params = new ArrayList<QueryParam>();
|
||||
params.add(new QueryParam("matchId", districtId));
|
||||
if(startDate!=null){
|
||||
params.add(new QueryParam("tradeDate", QueryParam.GREATER_THAN_EQUAL_OPERATOR, startDate));
|
||||
}
|
||||
if(endDate!=null){
|
||||
params.add(new QueryParam("tradeDate", QueryParam.LESS_THAN_EQUAL_OPERATOR, startDate));
|
||||
}
|
||||
|
||||
if (minPrice > 0) {
|
||||
params.add(new QueryParam("avgPrice", QueryParam.GREATER_THAN_OPERATOR, minPrice));
|
||||
}
|
||||
if (maxPrice > 0) {
|
||||
params.add(new QueryParam("avgPrice", QueryParam.LESS_THAN_OPERATOR, maxPrice));
|
||||
}
|
||||
List<SortParam> sortParams = new ArrayList<SortParam>();
|
||||
sortParams.add(new SortParam("tradeDate", false));
|
||||
List<DealPrice> dealPrices = dealPriceDao.findAllByParams(params, sortParams, limit);
|
||||
return dealPrices;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DealPrice> getDisSameDealPrice(DistrictKey key, int minPrice, int maxPrice, int limit) {
|
||||
List<QueryParam> params = new ArrayList<QueryParam>();
|
||||
|
||||
Reference in New Issue
Block a user