This commit is contained in:
wangwenyan@fangjia.com
2022-05-12 22:17:33 +08:00
80 changed files with 5064 additions and 242 deletions
@@ -0,0 +1,7 @@
package com.fangjia.dal.mongo.dao;
import com.fangjia.dal.MongoDao;
import com.fangjia.dal.mongo.model.BsEvaluation;
public interface BsEvaluationDao extends MongoDao<BsEvaluation> {
}
@@ -0,0 +1,8 @@
package com.fangjia.dal.mongo.dao;
import com.fangjia.dal.MongoDaoImpl;
import com.fangjia.dal.mongo.model.BsEvaluation;
public class BsEvaluationDaoImpl extends MongoDaoImpl<BsEvaluation> implements
BsEvaluationDao {
}
@@ -0,0 +1,184 @@
package com.fangjia.dal.mongo.model;
import com.fangjia.dal.MongoEntity;
import javax.persistence.*;
import java.util.Date;
@Entity
@Table(name = "business.evaluation")
public class BsEvaluation implements MongoEntity {
private String id;
private String fullAddress;
private String commid;
private String cityName;
private String distName;
private String commName;
private String address;
private Double buildArea;
private Integer floor;
private Integer floorcnt;
private Integer roomcnt;
private Integer livingroomcnt;
private String houseaspect;//ÄÏ
private String fitment;
private Date cDate;
private Integer unitPrice;
private Integer totalPrice;
@Id
@GeneratedValue(strategy = GenerationType.AUTO)
@Column(name = "_id")
public String getId() {
return id;
}
@Override
public void setId(String id) {
this.id = id;
}
@Column(name = "full_address")
public String getFullAddress() {
return fullAddress;
}
public void setFullAddress(String fullAddress) {
this.fullAddress = fullAddress;
}
@Column(name = "commid")
public String getCommid() {
return commid;
}
public void setCommid(String commid) {
this.commid = commid;
}
@Column(name = "cityname")
public String getCityName() {
return cityName;
}
public void setCityName(String cityName) {
this.cityName = cityName;
}
@Column(name = "distname")
public String getDistName() {
return distName;
}
public void setDistName(String distName) {
this.distName = distName;
}
@Column(name = "commname")
public String getCommName() {
return commName;
}
public void setCommName(String commName) {
this.commName = commName;
}
@Column(name = "address")
public String getAddress() {
return address;
}
public void setAddress(String address) {
this.address = address;
}
@Column(name = "buildarea")
public Double getBuildArea() {
return buildArea;
}
public void setBuildArea(Double buildArea) {
this.buildArea = buildArea;
}
@Column(name = "floor")
public Integer getFloor() {
return floor;
}
public void setFloor(Integer floor) {
this.floor = floor;
}
@Column(name = "floorcnt")
public Integer getFloorcnt() {
return floorcnt;
}
public void setFloorcnt(Integer floorcnt) {
this.floorcnt = floorcnt;
}
@Column(name = "roomcnt")
public Integer getRoomcnt() {
return roomcnt;
}
public void setRoomcnt(Integer roomcnt) {
this.roomcnt = roomcnt;
}
@Column(name = "livingroomcnt")
public Integer getLivingroomcnt() {
return livingroomcnt;
}
public void setLivingroomcnt(Integer livingroomcnt) {
this.livingroomcnt = livingroomcnt;
}
@Column(name = "houseaspect")
public String getHouseaspect() {
return houseaspect;
}
public void setHouseaspect(String houseaspect) {
this.houseaspect = houseaspect;
}
@Column(name = "fitment")
public String getFitment() {
return fitment;
}
public void setFitment(String fitment) {
this.fitment = fitment;
}
@Column(name = "c_date")
public Date getcDate() {
return cDate;
}
public void setcDate(Date cDate) {
this.cDate = cDate;
}
@Column(name = "unitprice")
public Integer getUnitPrice() {
return unitPrice;
}
public void setUnitPrice(Integer unitPrice) {
this.unitPrice = unitPrice;
}
@Column(name = "totalprice")
public Integer getTotalPrice() {
return totalPrice;
}
public void setTotalPrice(Integer totalPrice) {
this.totalPrice = totalPrice;
}
}
@@ -63,6 +63,7 @@ public class DistrictFacilitys implements MongoEntity {
private String imgUrl3D;
private List<String> school;
private List<String> keySchool;
@Id
@GeneratedValue
@@ -380,4 +381,12 @@ public class DistrictFacilitys implements MongoEntity {
this.school = school;
}
@Column(name = "key_school")
public List<String> getKeySchool() {
return keySchool;
}
public void setKeySchool(List<String> keySchool) {
this.keySchool = keySchool;
}
}
@@ -48,6 +48,11 @@ public class DealPrice implements MongoEntity {
private Integer floor;//当前楼层
private Long score;
private String matchId;
private String matchCity;
private String matchRegion;
private String matchName;
@Id
@GeneratedValue
@Column(name = "_id", unique = true, nullable = false)
@@ -296,4 +301,40 @@ public class DealPrice implements MongoEntity {
}
return true;
}
@Column(name = "match_id")
public String getMatchId() {
return matchId;
}
public void setMatchId(String matchId) {
this.matchId = matchId;
}
@Column(name = "match_city")
public String getMatchCity() {
return matchCity;
}
public void setMatchCity(String matchCity) {
this.matchCity = matchCity;
}
@Column(name = "match_region")
public String getMatchRegion() {
return matchRegion;
}
public void setMatchRegion(String matchRegion) {
this.matchRegion = matchRegion;
}
@Column(name = "match_name")
public String getMatchName() {
return matchName;
}
public void setMatchName(String matchName) {
this.matchName = matchName;
}
}
@@ -448,6 +448,7 @@ public class HouseLoudong extends com.fangjia.dal.mongo.model.HouseLoudong{
this.openTotalPrice = openTotalPrice;
}
@Column(name = "district_id")
public String getDistrictId() {
return districtId;
}
@@ -101,7 +101,11 @@ public class Loudong implements MongoEntity {
* seaweed中的小区名
*/
private String nameInSeaweed;
/**
* seaweed中的小区id
*/
private String idInSeaweed;
@Column(name = "remark")
public String getRemark() {
return remark;
@@ -402,4 +406,13 @@ public class Loudong implements MongoEntity {
public void setNameInSeaweed(String nameInSeaweed) {
this.nameInSeaweed = nameInSeaweed;
}
@Column(name = "id_in_seaweed")
public String getIdInSeaweed() {
return idInSeaweed;
}
public void setIdInSeaweed(String idInSeaweed) {
this.idInSeaweed = idInSeaweed;
}
}
@@ -67,6 +67,13 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
must.add(addTerms("district_name", alias));
must.add(addMatch("cat", "sell"));
must.add(addRange("avg_price", minPrice, maxPrice));
JSONObject term = new JSONObject();
JSONObject match = new JSONObject();
match.put("p_date", 20210501);
term.put("match", match);
must.add(term);
bool.put("must", must);
query.put("bool", bool);
reqBody.put("query", query);
@@ -58,6 +58,8 @@ public class Seawater implements SeawaterEntity {
private String url;
private String root;
private Integer avgPrice;
private String purpose;
private String estateType;
@Id
@Column(name = "_id")
@@ -285,4 +287,22 @@ public class Seawater implements SeawaterEntity {
public void setAvgPrice(Integer avgPrice) {
this.avgPrice = avgPrice;
}
@Column(name = "purpose")
public String getPurpose() {
return purpose;
}
public void setPurpose(String purpose) {
this.purpose = purpose;
}
@Column(name = "estate_type")
public String getEstateType() {
return estateType;
}
public void setEstateType(String estateType) {
this.estateType = estateType;
}
}
@@ -23,13 +23,13 @@ import java.util.regex.Pattern;
/**
* 法拍数据提供接口
*/
@Authority(role = Role.USER)
//@Authority(role = Role.USER)
public class AuctionController {
private PaiMaiService paiMaiService;
private DistrictNewPriceService districtNewPriceService;
private static final String[] SEARCH_FIELDS = new String[]{"marketAvgPrice", "startDate", "curPrice", "imgUrls", "id", "matchCity", "matchRegion", "matchName", "address", "marketTotalPrice", "startPrice", "curPrice", "predictPrice", "depositPrice", "area", "status", "url", "matchEstateType"};
private static final String[] SEARCH_FIELDS = new String[]{"marketAvgPrice", "startDate", "curPrice", "imgUrls", "id", "matchCity", "matchRegion", "matchName", "address", "marketTotalPrice", "startPrice", "curPrice", "predictPrice", "depositPrice", "area", "status", "url", "matchEstateType","type","count"};
public Object housesAction(AuctionModel model){
RestfulModelAndView mav = new RestfulModelAndView();
@@ -187,6 +187,27 @@ public class AuctionController {
return mav;
}
public Object searchPaimaisAction(AuctionModel model) {
RestfulModelAndView mav = new RestfulModelAndView();
List<QueryParam> params = new ArrayList<QueryParam>();
if(StringUtils.isNotBlank(model.getCity())){
params.add(new QueryParam("matchCity", model.getCity()));
}
if(StringUtils.isNotBlank(model.getDistrict())){
params.add(new QueryParam("matchRegion", model.getDistrict()));
}
if(StringUtils.isNotBlank(model.getName())){
params.add(new QueryParam("matchName", model.getName()));
}
List<SortParam> sortParams = new ArrayList<SortParam>();
sortParams.add(new SortParam("startDate", false));
List<PaiMai> paimais = paiMaiService.findAll(params, sortParams, model.getLimit(), SEARCH_FIELDS);
mav.setModel(BasicModel.buildOKModel().setResult(paimais));
return mav;
}
public void setPaiMaiService(PaiMaiService paiMaiService) {
this.paiMaiService = paiMaiService;
}
@@ -15,8 +15,10 @@ import java.util.Map.Entry;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.fangjia.dal.common.DistrictKey;
import com.fangjia.dal.mongo.model.Metro;
import com.fangjia.open.model.BasicModel;
import com.fangjia.service.bo.DistrictStatBo;
import org.apache.commons.lang.StringUtils;
import com.fangjia.core.util.DateUtil;
@@ -211,6 +213,42 @@ public class DealPriceController {
mv.setModel(resMap);
return mv;
}
public Object searchListAction(final DealPriceModel model) {
RestfulModelAndView mv = new RestfulModelAndView();
mv.setViewType("rest");
if (StringUtils.isBlank(model.getId())) {
mv.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "楼盘编号不能为空"));
return mv;
}
District district = districtService.findDistrictById(model.getId());
DistrictKey districtKey = new DistrictKey(district.getCity(), district.getRegion(), district.getName());
if(district == null){
mv.setModel(BasicModel.buildBadRequestModel("该小区不存在"));
return mv;
}
DistrictStatBo bo = districtStatService.findCurrentDistrictStat(districtKey);
Integer minPrice = null;
Integer maxPrice = null;
if(bo.getDistrictStat()!=null){
minPrice = (int)(bo.getDistrictStat().getAvgPrice() * 0.8);
maxPrice = (int)(bo.getDistrictStat().getAvgPrice() * 1.2);
}
Date startDate = null;
Date endDate = null;
if(StringUtils.isNotBlank(model.getStartDate())){
startDate = DateUtil.parseDate(model.getStartDate(),"yyyyMMdd");
}
if(StringUtils.isNotBlank(model.getEndDate())){
endDate = DateUtil.parseDate(model.getEndDate(),"yyyyMMdd");
}
List<DealPrice> dealPrices = dealPriceService.getNearlySameDealPrice(model.getId(), minPrice, maxPrice, startDate, endDate, model.getLimit());
mv.setModel(BasicModel.buildOKModel().setResult(dealPrices));
return mv;
}
private List<DealPrice> findDealPricesByName(String city,String region ,String name){
Calendar calendar = Calendar.getInstance();
@@ -7,11 +7,11 @@ import javax.servlet.http.HttpServletRequest;
import com.fangjia.core.RegionData;
import com.fangjia.core.util.DateUtil;
import com.fangjia.dal.mongo.model.RegionStat;
import com.fangjia.dal.mongo.model.RegionStatRent;
import com.fangjia.dal.mongo.model.*;
import com.fangjia.open.model.AvgPriceQueryModel;
import com.fangjia.open.model.BasicModel;
import com.fangjia.service.datastore.RegionStatService;
import com.fangjia.service.district.DistrictFacilitysService;
import com.fangjia.service.district.DistrictStatRentService;
import com.fangjia.service.district.RegionStatRentService;
import com.fangjia.service.mongo.fangjia.AverageAnalyService;
@@ -38,8 +38,6 @@ import com.fangjia.dal.common.DistrictKey;
import com.fangjia.dal.common.FangjiaServerAreaContextHolder;
import com.fangjia.dal.common.ServerAreaData;
import com.fangjia.dal.mongo.fangjia.model.ServerArea;
import com.fangjia.dal.mongo.model.DistrictHouse;
import com.fangjia.dal.mongo.model.DistrictStat;
import com.fangjia.dal.mongo.model.bkoff.ExtractDistrict;
import com.fangjia.dal.seaweed.model.District;
import com.fangjia.dal.sql.model.ApiUser;
@@ -93,6 +91,7 @@ public class DistrictController {
private RegionBlockService regionBlockService;
private RegionStatService regionStatService;
private RegionStatRentService regionStatRentService;
private DistrictFacilitysService districtFacilitysService;
public static final String[] COLUMNS = { "id", "name", "estateType", "buildType", "licence", "buildArea", "allArea",
@@ -444,19 +443,19 @@ public class DistrictController {
*
* @return
*/
public Object showAction(DistrictSearchModel model, @Param(name = "city") String city, @Param(name = "region") String region,
@Param(name = "name") String name, HttpServletRequest request) {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
if (StringUtils.isBlank(name) || StringUtils.isBlank(city) || StringUtils.isBlank(region)) {
mav.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "小区所在城市,小区名,所在区域必必须输入"));
return mav;
} else {
DistrictBo districtBo = districtService.findByFullMode(new DistrictKey(city, region, name));
if (null == districtBo) {
mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "小区数据不存在,请检查查询条件"));
public Object showAction(DistrictSearchModel model, @Param(name = "city") String city, @Param(name = "region") String region,
@Param(name = "name") String name, HttpServletRequest request) {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
if (StringUtils.isBlank(name) || StringUtils.isBlank(city) || StringUtils.isBlank(region)) {
mav.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "小区所在城市,小区名,所在区域必必须输入"));
return mav;
}
} else {
DistrictBo districtBo = districtService.findByFullMode(new DistrictKey(city, region, name));
if (null == districtBo) {
mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "小区数据不存在,请检查查询条件"));
return mav;
}
districtBo.setStat(this.districtStatService.findDistrictStat(districtBo.getDistrictKey()));
// 去除小区描述中的html标签
districtBo.setDescription(DataUtil.removeSpecialString(districtBo.getDescription()));
@@ -1146,6 +1145,25 @@ public class DistrictController {
ret.put("RESULT", 200);
return mav;
}
public Object facilitysAction(DistrictSearchModel model, @Param(name = "city") String city, @Param(name = "region") String region,
@Param(name = "name") String name, HttpServletRequest request) {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
if (StringUtils.isBlank(name) || StringUtils.isBlank(city) || StringUtils.isBlank(region)) {
mav.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "小区所在城市,小区名,所在区域必必须输入"));
return mav;
} else {
District districtBo = districtService.findDistrict(new DistrictKey(city, region, name), "district");
if (null == districtBo) {
mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "小区数据不存在"));
return mav;
}
DistrictFacilitys facilitys = districtFacilitysService.findByDistrict(districtBo);
mav.setModel(BasicModel.buildOKModel().setResult(facilitys));
}
return mav;
}
@@ -1230,4 +1248,7 @@ public class DistrictController {
this.regionStatRentService = regionStatRentService;
}
public void setDistrictFacilitysService(DistrictFacilitysService districtFacilitysService) {
this.districtFacilitysService = districtFacilitysService;
}
}
@@ -10,9 +10,14 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.fangjia.dal.mongo.model.BsEvaluation;
import com.fangjia.dal.mongo.model.bkoff.DealPrice;
import com.fangjia.service.bo.AddrEvaResult;
import com.fangjia.service.bo.DistrictStatBo;
import com.fangjia.service.bo.EvaluationBo;
import com.fangjia.service.bo.HouseScorer;
import com.fangjia.service.datastore.*;
import com.fangjia.service.district.DistrictStatService;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
@@ -54,11 +59,6 @@ import com.fangjia.open.model.RestfulModelAndView;
import com.fangjia.service.api.AutoCompleteService;
import com.fangjia.service.api.KnowbaseService;
import com.fangjia.service.common.SellSimilar;
import com.fangjia.service.datastore.BusinessEstateService;
import com.fangjia.service.datastore.DistrictService;
import com.fangjia.service.datastore.EvaluationReportService;
import com.fangjia.service.datastore.EvaluationService;
import com.fangjia.service.datastore.HouseService;
import com.fangjia.service.sql.ApiUserService;
import com.fangjia.service.sql.UserService;
import com.fangjia.service.web.AddressService;
@@ -94,6 +94,8 @@ public class EvaluationController {
private ApiUserService apiUserService;
private UserService userService;
private AddressService addressService;
private DistrictStatService districtStatService;
private BsEvaluationService bsEvaluationService;
private final static String AK = OpenConfig.getConfig().getBaiduAk();
public Object lngLatEvaluateAction(EvaluationModel model) {
@@ -1005,6 +1007,44 @@ public class EvaluationController {
return mav;
}
public Object bsListAction(EvaluationModel model) {
RestfulModelAndView mv = new RestfulModelAndView();
mv.setViewType("rest");
if (StringUtils.isBlank(model.getId())) {
mv.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "楼盘编号不能为空"));
return mv;
}
District district = districtService.findDistrictById(model.getId());
DistrictKey districtKey = new DistrictKey(district.getCity(), district.getRegion(), district.getName());
if(district == null){
mv.setModel(BasicModel.buildBadRequestModel("该小区不存在"));
return mv;
}
DistrictStatBo bo = districtStatService.findCurrentDistrictStat(districtKey);
Integer minPrice = null;
Integer maxPrice = null;
if(bo.getDistrictStat()!=null){
minPrice = (int)(bo.getDistrictStat().getAvgPrice() * 0.8);
maxPrice = (int)(bo.getDistrictStat().getAvgPrice() * 1.2);
}
Date startDate = null;
Date endDate = null;
if(StringUtils.isNotBlank(model.getStartDate())){
startDate = DateUtil.parseDate(model.getStartDate(),"yyyyMMdd");
}
if(StringUtils.isNotBlank(model.getEndDate())){
endDate = DateUtil.parseDate(model.getEndDate(),"yyyyMMdd");
}
List<BsEvaluation> dealPrices = bsEvaluationService.getNearlySameEvaluation(model.getId(), minPrice, maxPrice, startDate, endDate, model.getLimit());
mv.setModel(BasicModel.buildOKModel().setResult(dealPrices));
return mv;
}
public void setEvaluationService(EvaluationService evaluationService) {
this.evaluationService = evaluationService;
}
@@ -1017,6 +1057,18 @@ public class EvaluationController {
this.knowbaseService = knowbaseService;
}
public void setDistrictStatService(DistrictStatService districtStatService) {
this.districtStatService = districtStatService;
}
public void setBsEvaluationService(BsEvaluationService bsEvaluationService) {
this.bsEvaluationService = bsEvaluationService;
}
public void setAutoCompleteService(AutoCompleteService autoCompleteService) {
this.autoCompleteService = autoCompleteService;
}
@@ -11,6 +11,8 @@ import com.fangjia.dal.mongo.model.DistrictFacilitys;
import com.fangjia.dal.mongo.model.FangjiaOfficialAreaMap;
import com.fangjia.dal.mongo.model.Metro;
import com.fangjia.dal.mongo.model.StatsIndex;
import com.fangjia.dal.seawater.dao.SeawaterDao;
import com.fangjia.dal.seawater.model.Seawater;
import com.fangjia.dal.seaweed.model.District;
import com.fangjia.dal.sql.model.ApiUser;
import com.fangjia.datastore.driver.FacetResult;
@@ -29,6 +31,7 @@ import com.fangjia.open.ws.query.CollateralSampleQuery;
import com.fangjia.open.ws.query.ShopCollateralSampleQuery;
import com.fangjia.service.address.AddressParser;
import com.fangjia.service.bo.Collateral;
import com.fangjia.service.bo.DistrictStatBo;
import com.fangjia.service.bo.Sample;
import com.fangjia.service.bo.ShopSample;
import com.fangjia.service.datastore.CzbankSampleService;
@@ -39,11 +42,13 @@ import com.fangjia.service.datastore.bean.CollateralSampleQueryBean;
import com.fangjia.service.datastore.bean.ShopCollateralSampleQueryBean;
import com.fangjia.service.district.DistrictFacilitysService;
import com.fangjia.service.district.DistrictStatService;
import com.fangjia.service.district.SqmetreRentService;
import com.fangjia.service.menu.Item;
import com.fangjia.service.menu.MenuCategoryEnum;
import com.fangjia.service.menu.Value;
import com.fangjia.service.mongo.fangjia.DealPriceService;
import com.fangjia.service.mongo.fangjia.MetroService;
import com.fangjia.service.mongo.fangjia.SeawaterService;
import com.fangjia.service.mongo.fangjia.StatsIndexService;
import com.fangjia.service.open.FangjiaOfficialAreaService;
import com.fangjia.service.open.HousePojo;
@@ -86,6 +91,9 @@ public class HouseController {
private CzbankSampleService czbankSampleService;
private DistrictService districtService;
private FangjiaOfficialAreaService fangjiaOfficialAreaService;
private SeawaterService seawaterService;
private SeawaterDao seawaterDao;
private List<String> PROPERTY_TYPE_CODE_LIST = Arrays.asList("01", "02");
@@ -399,6 +407,49 @@ public class HouseController {
}
}
public Object listAction(final HouseSearchModel model) {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
if (StringUtils.isBlank(model.getId()) && (StringUtils.isBlank(model.getCity()) || StringUtils.isBlank(model.getRegion()) || StringUtils.isBlank(model.getName()))) {
mav.setModel(BasicModel.buildBadRequestModel("楼盘编号或城市,区域,名称为必填项"));
return mav;
}
District district = null;
DistrictKey districtKey = null;
if(StringUtils.isNotBlank(model.getName())){
districtKey = new DistrictKey(model.getCity(), model.getRegion(), model.getName());
district = districtService.findDistrict(districtKey, "district");
}else{
district = districtService.findDistrictById(model.getId());
districtKey = new DistrictKey(district.getCity(), district.getRegion(), district.getName());
}
if(district == null){
mav.setModel(BasicModel.buildBadRequestModel("该小区不存在"));
return mav;
}
DistrictStatBo bo = districtStatService.findCurrentDistrictStat(districtKey);
Integer minPrice = null;
Integer maxPrice = null;
if(bo.getDistrictStat()!=null){
minPrice = (int)(bo.getDistrictStat().getAvgPrice() * 0.8);
maxPrice = (int)(bo.getDistrictStat().getAvgPrice() * 1.2);
}
List<String> names = new ArrayList<String>();
names.add(district.getName());
if(district.getAlias() != null){
for(String alia : district.getAlias()){
if(StringUtils.isNotBlank(alia) && !names.contains(alia)){
names.add(alia);
}
}
}
List<Seawater> houses = seawaterDao.findSeawatersFromEs("sell", district.getCity(), district.getRegion(), district.getName(), district.getAlias(), null, model.getStartTime(), model.getEndTime(),null,null,minPrice, maxPrice,null,null,"p_date", false,0, model.getPageSize(),"latest_seawater");
//List<Seawater> houses1 = seawaterDao.findHouses(district.getCity(), district.getRegion(), names, minPrice, maxPrice, model.getPageSize());
mav.setModel(BasicModel.buildOKModel().setResult(houses));
return mav;
}
String getPicUrl(String url, List<String> urls) {
if (!urls.contains(url)) {
urls.add(url);
@@ -832,6 +883,13 @@ public class HouseController {
return ret;
}
public void setSeawaterService(SeawaterService seawaterService) {
this.seawaterService = seawaterService;
}
public void setSeawaterDao(SeawaterDao seawaterDao) {
this.seawaterDao = seawaterDao;
}
private boolean checkExists(FacetQueryResult<HousePojo> houseBoList, HousePojo house) {
for (HousePojo bo : houseBoList) {
@@ -665,6 +665,7 @@ public class PropertyController {
return mav;
}
String userId = FangjiaApiUtil.parseUserId(model.getToken());
District district = null;
if (StringUtils.isNotBlank(model.getId())){
if(ObjectId.isValid(model.getId())){
@@ -728,7 +729,7 @@ public class PropertyController {
} else {
districtInfoDto.setBuildingType("");
}
districtInfoDto.setAlias(district.getAlias());
districtInfoDto.setAlias(district.getMappingAlias());
districtInfoDto.setAddressAlias(district.getAddrAlias());
districtInfoDto.setCommunity(district.getStreetCommunity());
districtInfoDto.setCommercialArea(district.getBusinessQuarter());
@@ -780,6 +781,7 @@ public class PropertyController {
districtInfoDto.setDescription(district.getDescription());
districtInfoDto.setSalesOfficeAdd(district.getSellAddress());
districtInfoDto.setSalesPhoneNumber(district.getMobile());
districtInfoDto.setTownship(district.getTownship());
if (district.getLoudongCount() != null && district.getLoudongCount() > 0) {
districtInfoDto.setBuildingCount(district.getLoudongCount());
}
@@ -948,6 +950,12 @@ public class PropertyController {
return mav;
}
if(model.getMaxFloor() == null && model.getFloor() != null &&StringUtils.isNotBlank(model.getBuildingNumber())){
Loudong loudong = loudongService.findForEvaluate(district.getId(), model.getBuildingNumber(), model.getUnitNumber());
if(loudong != null && loudong.getFloors() >= model.getFloor()){
model.setMaxFloor(loudong.getFloors());
}
}
model.setRegion(district.getRegion());
model.setDistrictName(district.getName());
if (StringUtils.isBlank(model.getPropertyType())) {
@@ -1104,7 +1112,7 @@ public class PropertyController {
model.setCalDate(calDate);
}
/* 先到CPDB库中查找小区 */
District district = matchDistrict(model.getCity(), model.getAddress(), model.getDistrict(), model.getName());
District district = districtService.findDistrict(new DistrictKey(model.getCity(), model.getDistrict(), model.getName()), "district");//matchDistrict(model.getCity(), model.getAddress(), model.getDistrict(), model.getName());
if(district == null){
//防止地址模糊,名称详细的情况
district = matchDistrict(model.getCity(), model.getName(), model.getDistrict(), model.getAddress());
@@ -5400,6 +5408,36 @@ public class PropertyController {
}
}
public Object searchHouseLoudongAction(PropertyEvaluationModel model, HttpServletRequest request)throws NumberFormatException, IllegalAccessException {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
if (StringUtils.isBlank(model.getId()) || StringUtils.isBlank(model.getHouseNumber()) || StringUtils.isBlank(model.getRoomNumber())){
mav.setModel(BasicModel.buildBadRequestModel("参数非法:楼盘编号,楼栋号,室为必填项"));
return mav;
}
List<HouseLoudong> houseLoudongs = houseLoudongService.findAll(model.getId(), model.getHouseNumber(), model.getUnitNumber(), model.getRoomNumber());
mav.setModel(BasicModel.buildOKModel().setResult(houseLoudongs));
return mav;
}
public Object searchLoudongAction(PropertyEvaluationModel model, HttpServletRequest request)throws NumberFormatException, IllegalAccessException {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
if (StringUtils.isBlank(model.getId()) || StringUtils.isBlank(model.getHouseNumber())){
mav.setModel(BasicModel.buildBadRequestModel("参数非法:楼盘编号,楼栋号为必填项"));
return mav;
}
List<Loudong> loudongs = loudongService.findAll(model.getId(), model.getHouseNumber(), model.getUnitNumber());
mav.setModel(BasicModel.buildOKModel().setResult(loudongs));
return mav;
}
public void setHouseLoudongService(HouseLoudongService houseLoudongService) {
this.houseLoudongService = houseLoudongService;
@@ -0,0 +1,36 @@
package com.fangjia.open.controllers;
import com.fangjia.core.util.AnalyticsUtil;
import com.fangjia.open.interceptors.Authority;
import com.fangjia.open.interceptors.Role;
import com.fangjia.open.model.BasicModel;
import com.fangjia.open.model.RestfulModelAndView;
import javax.servlet.http.HttpServletRequest;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
public class requestController {
/**
* »ñÈ¡ÇëÇóµÄrequestºÍIPÐÅÏ¢
* @param request
* @return
*/
public Object requestAction(HttpServletRequest request) {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
String ip = AnalyticsUtil.getIpAddress(request);
Map<String, String> map = new HashMap<String, String>();
Enumeration headerNames = request.getHeaderNames();
while (headerNames.hasMoreElements()) {
String key = (String) headerNames.nextElement();
String value = request.getHeader(key);
map.put(key, value);
}
map.put("ip", ip);
mav.setModel(BasicModel.buildOKModel().setResult(map));
return mav;
}
}
@@ -99,6 +99,7 @@ public class DistrictInfoDto {
private String salesPhoneNumber;
private Integer buildingCount;
private Integer finishYear;
private String township;
public String getDistrict() {
@@ -818,4 +819,12 @@ public class DistrictInfoDto {
public void setFinishYear(Integer finishYear) {
this.finishYear = finishYear;
}
public String getTownship() {
return township;
}
public void setTownship(String township) {
this.township = township;
}
}
@@ -8,6 +8,7 @@ public class AuctionModel extends FangjiaApiModel{
private String city;
private String district;
private String address;
private String name;
private Date minStartTime;
private Date maxStartTime;
private Integer minStartPrice;
@@ -121,4 +122,12 @@ public class AuctionModel extends FangjiaApiModel{
public void setStatus(Integer status) {
this.status = status;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
}
@@ -1,6 +1,9 @@
package com.fangjia.open.model;
import java.util.Date;
public class DealPriceModel extends FangjiaApiModel{
private String id;
private String city;
@@ -34,9 +37,13 @@ public class DealPriceModel extends FangjiaApiModel{
private Double size;
private Integer max;
private Integer limit = 10;
private String decoration;
private boolean convert;
private String startDate;
private String endDate;
public String getCity() {
@@ -198,4 +205,36 @@ public class DealPriceModel extends FangjiaApiModel{
public void setConvert(boolean convert) {
this.convert = convert;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getStartDate() {
return startDate;
}
public void setStartDate(String startDate) {
this.startDate = startDate;
}
public String getEndDate() {
return endDate;
}
public void setEndDate(String endDate) {
this.endDate = endDate;
}
public Integer getLimit() {
return limit;
}
public void setLimit(Integer limit) {
this.limit = limit;
}
}
@@ -246,6 +246,42 @@ public class EvaluationModel extends FangjiaApiModel {
private Date buildDate;
private String sid;//微信小程序中的sid
private String code;//微信小程序中的code
private String id;
private String startDate;
private String endDate;
private Integer limit = 10;
public Integer getLimit() {
return limit;
}
public void setLimit(Integer limit) {
this.limit = limit;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getStartDate() {
return startDate;
}
public void setStartDate(String startDate) {
this.startDate = startDate;
}
public String getEndDate() {
return endDate;
}
public void setEndDate(String endDate) {
this.endDate = endDate;
}
public String getRegion() {
return region;
@@ -4,6 +4,7 @@ import com.fangjia.service.datastore.bean.HouseQueryBean;
import com.fangjia.service.query.QueryBean;
public class HouseSearchModel extends SearchModel {
private String id;
private String kind;
private String estateType;
private String buildingType;
@@ -116,7 +117,12 @@ public class HouseSearchModel extends SearchModel {
public void setShowDetail(Boolean showDetail) {
this.showDetail = showDetail;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
}
@@ -46,7 +46,12 @@ public class PropertyEvaluationModel extends FangjiaApiModel {
* Â¥ºÅ
*/
private String unitNumber;
/**
* ÐÂÂ¥ºÅ
*/
private String houseNumber;
private String id;
/**
* ÊÒºÅ
*/
@@ -905,4 +910,20 @@ public class PropertyEvaluationModel extends FangjiaApiModel {
public void setAvgPrice(Integer avgPrice) {
this.avgPrice = avgPrice;
}
public String getHouseNumber() {
return houseNumber;
}
public void setHouseNumber(String houseNumber) {
this.houseNumber = houseNumber;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
}
@@ -16,6 +16,8 @@ public abstract class SearchModel extends FangjiaApiModel {
protected Integer totalPriceEnd;
protected Integer avgPriceStart;
protected Integer avgPriceEnd;
protected Integer startTime;
protected Integer endTime;
protected Integer areaStart;
protected Integer areaEnd;
protected Integer room;
@@ -358,4 +360,20 @@ public abstract class SearchModel extends FangjiaApiModel {
public void setTotalPrice(Integer totalPrice) {
this.totalPrice = totalPrice;
}
public Integer getStartTime() {
return startTime;
}
public void setStartTime(Integer startTime) {
this.startTime = startTime;
}
public Integer getEndTime() {
return endTime;
}
public void setEndTime(Integer endTime) {
this.endTime = endTime;
}
}
@@ -68,20 +68,30 @@ class BigdataBuildingBatch extends BaseBatch{
def q = new BasicDBObject()
q.put("city", city)
q.put("status", 1)
// q.put("name","ÁÉÔ´Ò»´å")
def dbcursor = LOU_DONG_STAT.find(q)
def map = [:]
def set = new HashSet()
for(def db : dbcursor){
def list = map.get(db.region+"##"+db.name)
if(!list){
list = []
map.put(db.region+"##"+db.name, list)
}
list << db
if(db.house_num && db.house_num!="--"){
def key = db.region+"##"+db.name +"##"+db.house_num
if(!set.contains(key)){
set << key
list << db
}
}
}
return map
}
def createCityDatas(cityCodeDatas, distCodeMap, distMap, buildingMap){
def rets = []
def keys = []
distMap.each{k,dist->
def buildings = buildingMap.get(k)
if(buildings){
@@ -89,53 +99,57 @@ class BigdataBuildingBatch extends BaseBatch{
def ret = []
def areaData = distCodeMap.get(obj.region)
if(areaData){
ret << cityCodeDatas.get(obj.city)
ret << obj._id.toString()
ret << obj.house_num
ret << ""
ret << ""
ret << areaData.district_code
ret << dist._id.toString()
if(obj.build_year){
ret << obj.build_year
}else{
def key = dist._id.toString() +"##"+obj.house_num
if(!keys.contains(key)){
keys << key
ret << cityCodeDatas.get(obj.city)
ret << obj._id.toString()
ret << obj.house_num
ret << obj.house_num
ret << ""
}
def arrs = CcbCodeUtils.parseBuildType(dist.build_type)
if(arrs){
ret << arrs[0].trim()
}else{
ret << ''
}
ret << areaData.district_code
ret << dist._id.toString()
if(obj.build_year){
ret << obj.build_year
}else{
ret << ""
}
def arrs = CcbCodeUtils.parseBuildType(dist.build_type)
if(arrs){
ret << arrs[0].trim()
}else{
ret << ''
}
ret << CcbCodeUtils.getBuildStructureCode(dist.build_structure)
if(obj.floors){
ret << obj.floors
}else{
ret << ""
}
ret << CcbCodeUtils.getBuildStructureCode(dist.build_structure)
if(obj.floors){
ret << obj.floors
}else{
ret << ""
}
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
// if(obj.elevators && obj.elevators>=1){
// ret << "1"
// }else{
// ret << "0"
// }
ret << ""
ret << ""
// if(obj.elevators && obj.elevators>=1) {
// ret << obj.elevators//µçÌÝÊý
// }else{
// ret << ""
// }
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << "1"
rets << ret
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << "1"
rets << ret
}
}
}
}
@@ -367,7 +367,7 @@ class CcbCommonUtils {
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
// q.put("name","特房锦绣一方")
// q.put("name","辽源一村")
// q.put("name_cat", 0);
// q.put("estate_type2", new BasicDBObject('$in', ["ĆŐͨץՏ","ÉĚץ"]))
@@ -57,6 +57,12 @@ class CcbFtpUtils {
ftpUpload(dataCodeFileName)//上传加密码文件到FTP
}
static def listFileNames(month){
//上传到ftp
sftpChannel = reConnect();
Vector files = sftp.listFiles(ftpPath+ month+"/", sftpChannel);
return files
}
static main(args) {
@@ -25,8 +25,8 @@ class DealPriceDataProcess {
def task = new DealPriceDataProcess()
// task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据
// task.addLianJiaPrice();
// task.deleteThirdPartyData()
task.delDealByPe()
task.deleteThirdPartyData()
// task.delDealByPe()
}
def delDealByPe(){
@@ -370,26 +370,26 @@ class DealPriceDataProcess {
def deleteThirdPartyData(){
def citys = ["东莞"] //,"杭州""广州",
def citys = CcbCommonUtils.citys
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def publishTime = '2021-10-01'
def month = 202110
def month = 202203
def rows = []
for(def city : citys) {
def rets = findPeGteThirdParty(city, 202111)
def rets = findPeGteThirdParty(city, 202204)
def total = 0
for(def m : rets){
def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
sqlInstance.executeUpdate(sql)
sqlInstance236.executeUpdate(sql)
}
println city +" "+total
println city +" "+rets.size()
}
}
def findPeGteThirdParty(cityName, yearmonth){
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>20"
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='µÚÈý·½' and cityname='${cityName}' and pe>50"
def rets = []
sqlInstance2.eachRow(sql,{r->
sqlInstance236.eachRow(sql,{r->
def map = [:]
map.commid = r.commid
map.price = r.target_avg_price
@@ -38,8 +38,8 @@ class ExportOracleData {
exportCommunityBuileType()
exportThirdPartyPrice2(lastMonth)
exportBigdataForsale("2022-02-19")
exportBigdataDeal("2022-02-01")
exportBigdataForsale("2022-03-19")
exportBigdataDeal("2022-03-01")
//计算结果相关
@@ -3,8 +3,10 @@ package com.fangjia.script.ccbevaluation
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.jcraft.jsch.ChannelSftp
import groovy.sql.Sql
import com.fangjia.script.ccbevaluation.ronghe.SSHHelper
import org.apache.commons.codec.digest.DigestUtils
class ExportOracleEvaluationData {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
@@ -33,6 +35,8 @@ class ExportOracleEvaluationData {
exportCommCaseDetailGj(curMonth)
def curDate = curMonth.toString()+"28"
// 生成MD5文件
exportMd5()
SSHHelper jschCommand = new SSHHelper();
System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
// okÎļþ
@@ -413,4 +417,23 @@ class ExportOracleEvaluationData {
CcbFtpUtils.ftpUpload(null, dataCodeFileName, curDate)
}
def exportMd5(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
Vector files = CcbFtpUtils.listFileNames(curDate)
Iterator fileVec = files.iterator()
def fileRows = []
while(fileVec.hasNext()){
ChannelSftp.LsEntry file = fileVec.next()
if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){
fileRows << file.getFilename() +":"+ DigestUtils.md5Hex(file.getFilename())
}
}
def fileName = "/ccb/md5.txt"
if(CommonDatautils.isWindow()){
fileName = "D:\\房价网\\md5.txt"
}
FileUtils.createTxtFile(fileRows, fileName, "utf-8", false);
CcbFtpUtils.ftpUpload(null, fileName, curDate)
println "md5.txt 上传Ftp完成......"
}
}
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
@BatchOwner(Watcher.GaoJuan)
class ThirdPartyPriceBatch extends BaseBatch {
def sDate= 202202//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202112
def endDate = 202202
def sDate= 202203//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202201
def endDate = 202203
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
@@ -75,13 +75,13 @@ class TradeStatLog {
def run(){
def citys = CcbCommonUtils.citys;
def rets = []
rets << ["城市","12月总量","1月总量","2月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
rets << ["城市","1月总量","2月总量","3月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2021-12-01"
def date = "2022-01-01"
def endDate = "2022-02-01"
def newDate = "2022-03-01"
def startDate = "2022-01-01"
def date = "2022-02-01"
def endDate = "2022-03-01"
def newDate = "2022-04-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -131,7 +131,7 @@ class TradeStatLog {
}
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-02成交统计.xlsx", rets, "数据")
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-03成交统计.xlsx", rets, "数据")
println "表格生成完成"
}
@@ -481,8 +481,8 @@ class TradeStatLog {
def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"]
def rets = []
rets << headers
def startDate = "2022-02-01"
def endDate = "2022-03-01"
def startDate = "2022-03-01"
def endDate = "2022-04-01"
for(def city : citys){
def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
@@ -2,25 +2,243 @@ package com.fangjia.script.ccbevaluation.data
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.mongodb.BasicDBObject
import java.util.concurrent.CopyOnWriteArrayList
class CcbDistrictToDB {
static main(args) {
def task = new CcbDistrictToDB();
// task.run()
task.run1()
// task.run1()
// task.meituanRun()
// task.meituanSqrentRun()
// task.updateDistrict()
task.updateDongguangRegion()
}
def updateDongguangRegion(){
def cDate = DateUtils.parseFromTightByPattern("2022-04-27","yyyy-MM-dd")
def dist = CommonDatautils.findAllDistrictByCity("东莞", "district", ["city","region","name","c_date","estate_type2"])
for(def d : dist){
def t = d.c_date
if(t.getTime()>cDate.getTime()){
}
}
}
def updateDistrict(){
def citys = ["惠州","东莞","金华","苏州","桂林","潮州","廊坊","临沧","通辽","雅安"]
def cDate = DateUtils.parseFromTightByPattern("2022-04-27","yyyy-MM-dd")
def index = 0
for(def city : citys){
def dist = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","c_date","estate_type2"])
for(def d : dist){
def t = d.c_date
if(t.getTime()>cDate.getTime()){
if(d.name.contains("(")){
def delstr = d.name.substring(d.name.indexOf("("), d.name.indexOf(")")+1)
def newName = d.name.replace(delstr,"")
index++
println index+" "+d.city+" "+d.name+" "+newName
def obj = CommonDatautils.findDistrict(d.city, d.region, newName, "district")
if(!obj){
println "update"
def udata = new BasicDBObject()
udata.put("name", newName)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
d.name = newName
addPrice(d)
def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1));
CommonUtils.recalculationSqRent(d.city, d.region, newName, d.estate_type2, lastMonthDay,false,0)
}else{
println "delete"
DbUtils.seaweed.remove(new BasicDBObject("_id",d._id))
}
}
}
}
}
}
def meituanSqrentRun(){
ExcelUtils.modify("D:\\数据\\美团\\美团需入库小区合并后.xlsx", { d ->
def city = d['F']
def region = d['G']
def newRegion = RegionData.formatRegion(city, region)
if(newRegion) {
def name = d['H']
name = nameFileter(name)
if(name){
def estateType2 = "普通住宅"
if(name.contains("别墅")){
estateType2 = "别墅"
}else if(name.contains("商住")){
estateType2 = "商住"
}else{
def type = d['I']
if(type){
if(type.contains("别墅")){
estateType2 = "别墅"
}else if(type.contains("商住")){
estateType2 = "商住"
}
}
}
def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1));
CommonUtils.recalculationSqRent(city, region, name, estateType2, lastMonthDay,false,0)
}
}
},0);
}
def meituanRun(){
def index = 0//美团需入库小区合并后.xlsx
def keys = new CopyOnWriteArrayList()
ExcelUtils.multiThreadingModify("D:\\数据\\美团\\美团需入库小区合并后.xlsx", { d ->
def city = d['F']
def region = d['G']
def newRegion = RegionData.formatRegion(city, region)
if(newRegion){
def name = d['H']
name = nameFileter(name)
def formatAddress = null
if(d['E']){
formatAddress = AddressUtil.formatAddress(city, d['E'])
}
if(!formatAddress || (formatAddress && !formatAddress.contains("号") && !formatAddress.contains("交叉"))){
def rets = OpenApiUtils.getGaoDeSearchLocation(city, region, name)
def map = findStandardObject2(city,region, name, d['E'], rets);
if(map && map.address){
formatAddress = AddressUtil.formatAddress(city, map.address)
if(!formatAddress){
if(map.address.contains("交叉口")){
formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉口")+3)
}else if(map.address.contains("交叉")){
formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉")+2)
}
}
}
}
Long glat = null
Long glng = null
Long blat = null
Long blng = null
if(d['K']){
glat = d['L'].toString().toDouble() * 10000000000L
glng = d['K'].toString().toDouble() * 10000000000L
def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3)
blng = baidu.x * 10000000000L
blat = baidu.y * 10000000000L
}
def estateType2 = "普通住宅"
if(name.contains("别墅")){
estateType2 = "别墅"
}else if(name.contains("商住")){
estateType2 = "商住"
}else{
def type = d['I']
if(type){
if(type.contains("别墅")){
estateType2 = "别墅"
}else if(type.contains("商住")){
estateType2 = "商住"
}
}
}
def key = city+"##"+newRegion+"##"+name
if(name && !keys.contains(key)){
def dist = CommonDatautils.findDistrict(city, newRegion, name, "district")
if(!dist){
keys << key
def data = new BasicDBObject();
data.put("name_cat",0)
data.put("status",0)
data.put("cat","district")
data.put("c_date", new Date())
data.put("m_date", new Date())
data.put("city", city)
data.put("region",newRegion)
data.put("name", name)
data.put("estate_type2",estateType2)
data.put("address", formatAddress)
if(glat){
data.put("lat",glat)
data.put("lng",glng)
data.put("lat2",blat)
data.put("lng2",blng)
}
def markMap = [:]
markMap.put("data_source", "机器入库")
data.put("mark_map",markMap)
index++
DbUtils.seaweed.save(data);
println index+" "+data
def stop = false
while (!stop){
if(data._id.toString()){
addPrice(data)
stop = true;
}
}
}
}
}else{
println city +" "+region +" 区域无法标准化"
}
},0);
}
def addPrice(dist){
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-03-01")
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-25")
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
for(def weekend : weekends){
CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0)
}
}
def run1(){
def index = 0
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\新增小区数据_1236.xlsx", { d ->
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\新增小区数据_1236.xlsx", { d ->
def city = d['A']
def region = d['B']
def newRegion = RegionData.formatRegion(city, region)
@@ -71,6 +289,7 @@ class CcbDistrictToDB {
if(name){
def dist = CommonDatautils.findDistrict(d['A'], newRegion, name, "district")
if(!dist){
def data = new BasicDBObject();
data.put("name_cat",0)
data.put("status",0)
@@ -221,4 +440,77 @@ class CcbDistrictToDB {
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
return name;
}
def findStandardObject2(def city, def region, def name, def address, def rets) {
if (rets) {
def bestObj = [:]
if (rets.size() >= 1) {
for (def mapObj : rets) {
def score = 0
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
score = 0.2
}
if(mapObj.name && name){
def newName = mapObj.name;
def names = []
if(mapObj.name && mapObj.name.indexOf("(")>-1){
def name1 = mapObj.name.substring(0, mapObj.name.indexOf("("))
names << name1
def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")"))
names << name2
}else{
names << newName
}
def maxNameScore = 0
for(def n : names){
double score1 = CosineSimilarAlgorithm.getSimilarity(n, name);
if(score1 && score1>0){
def nameScore = (score1 * 4)/10
if(nameScore > maxNameScore){
maxNameScore = nameScore
}
}
}
if(maxNameScore){
try{
maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore)
}catch(Exception e){
maxNameScore = 0
}
}
score = maxNameScore + score;
}
if(mapObj.address && address){
double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address);
if(addressScore && addressScore>0){
try{
addressScore = DataHandleUtils.getDoubleRound(2, addressScore)
}catch(Exception e){
addressScore = 0
}
score = addressScore*4/10 + score
}
}
if(!bestObj){
mapObj.score = score
bestObj = mapObj
}else{
if(score > bestObj.score){
mapObj.score = score
bestObj = mapObj
}
}
}
}
return bestObj
}
return null
}
}
@@ -16,7 +16,9 @@ import com.mongodb.DBObject
import groovy.sql.Sql
import org.apache.lucene.queryparser.classic.QueryParser
/**
* 重庆估值测试--2小时
*/
class EvaluationBatch {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
@@ -27,8 +29,8 @@ class EvaluationBatch {
static main(args) {
def task = new EvaluationBatch()
// task.run()
task.run1();
// task.run2()
// task.run1();
task.run2()
}
def run2(){
@@ -8,42 +8,17 @@ import com.fangjia.script.util.OpenApiClient
class BlankMatch {
static main(args) {
def task = new BlankMatch()
// task.run()
task.run()
// task.run1()
// task.run2()
task.run3()
}
def run3(){
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row ->
if(!row['C']){
def dist = CommonDatautils.findDistrict("重庆", row['E'], row['F'], "district")
if(dist){
row['C'] = dist._id.toString()
}
}
},0)
}
def run2(){
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配不上(合并).xlsx",0,1)
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row ->
if(!row['C']){
for(def d : datas){
if(d['B'] == row['B']){
row['C'] = d['C']
row['D'] = d['D']
row['E'] = d['E']
row['F'] = d['J']
break;
}
}
}
},0)
// task.run3()
}
def run() {
//rMap = SeaweedDataMatchProcess.doMatch(city, region, d['G'], null);
ExcelUtils.modify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row ->
//def path = "D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx";
def path = "D:\\数据\\重庆农商行\\2022-04-02\\测试.xlsx"
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['C']){
def jsonObject = OpenApiClient.addressMatch("ÖŘÇě", row['B'], "property", false, null, null)
if(jsonObject != null && jsonObject.precise!=-1) {
@@ -54,8 +29,6 @@ class BlankMatch {
row['G'] = jsonObject.get("address")
}
}
});
}
@@ -81,4 +54,31 @@ class BlankMatch {
// }
});
}
def run3(){
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row ->
if(!row['C']){
def dist = CommonDatautils.findDistrict("重庆", row['E'], row['F'], "district")
if(dist){
row['C'] = dist._id.toString()
}
}
},0)
}
def run2(){
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配不上(合并).xlsx",0,1)
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row ->
if(!row['C']){
for(def d : datas){
if(d['B'] == row['B']){
row['C'] = d['C']
row['D'] = d['D']
row['E'] = d['E']
row['F'] = d['J']
break;
}
}
}
},0)
}
}
@@ -0,0 +1,254 @@
package com.fangjia.script.chongqing
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.gj.paimai.DataUtil
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import groovy.sql.Sql
import java.util.concurrent.CopyOnWriteArrayList
class MatchEvaluateRegionTest {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx"
def total = 1500
static main(args) {
def task = new MatchEvaluateRegionTest()
task.run()//小区匹配
task.run1()//地图匹配
// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除
task.addPrice()
task.addArroundPrice()
task.addAuthor()
}
def run(){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['F']){
def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], "property", false, null, null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['F'] = jsonObject.get("id")
row['G'] = jsonObject.get("city")
row['H'] = jsonObject.get("district")
row['I'] = jsonObject.get("name")
row['J'] = jsonObject.get("address")
row['K'] = jsonObject.get("propertyType")
row['L'] = jsonObject.get("map")
// row['K'] = jsonObject.get("lng")
// row['L'] = jsonObject.get("lat")
}
}
});
}
def run1(){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['F']){
def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], null, true, null, null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['F'] = jsonObject.get("id")
row['G'] = jsonObject.get("city")
row['H'] = jsonObject.get("district")
row['I'] = jsonObject.get("name")
row['J'] = jsonObject.get("address")
row['K'] = jsonObject.get("propertyType")
row['L'] = jsonObject.get("map")
// row['K'] = jsonObject.get("lng")
// row['L'] = jsonObject.get("lat")
}
}
});
}
def addPrice(){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['L']){
def priceObj = CommonDatautils.findDistrictPrice(row['G'], row['H'], row['I'])
if(priceObj){
row['M'] = priceObj.avg_price
row['N']= (int)(priceObj.avg_price * row['E'])
}
def avg_price = findAvgPrice(row['M'])
if(row['M'] && avg_price){
def weekendPrice = row['M'].toString().toDouble()
def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
row['O'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15)
}
}
});
}
def addArroundPrice(){
ExcelUtils.multiThreadingModify(path, { row ->
if(row['A'] && !row['K']) {
def arroundObj = FangjiaClientUtils.arroundPrice(row['A'], row['B'])
if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
def priceObj = arroundObj.result
row['M'] = priceObj.unitPrice
row['N'] = (int) (priceObj.unitPrice * row['C'])
row['Q'] = "周边价格"
}
}
});
}
def addAuthor(){
def authors = ["丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"]
def total = 747
def personTotal = authors.size()
def avg = Math.ceil(total/personTotal)
def index = 0
def count = 0;
ExcelUtils.modify(path, { row ->
if(index < avg){
row['P'] = authors[count]
}
index++
if(index >= avg){
count++
index = 0
}
});
}
def addDistrict(){
def path = "D:\\数据\\重庆农商行\\2022-05-06\\需新增小区.xlsx"
def keys = new CopyOnWriteArrayList()
def index = 0
ExcelUtils.modify(path, { d->
def flag = d['M']
if(!flag){
def city = d['E']
def region = d['F']
def newRegion = RegionData.formatRegion(city, region)
if(newRegion){
def name = d['G']
name = nameFileter(name)
def formatAddress = null
def address = d['H']
if(address){
formatAddress = AddressUtil.formatAddress(city, address)
if(!formatAddress){
if(address.contains("交叉口")){
formatAddress = address.substring(0, address.toString().indexOf("交叉口")+3)
}else if(address.contains("交叉")){
formatAddress = address.substring(0, address.toString().indexOf("交叉")+2)
}
}
}
Long glat = null
Long glng = null
Long blat = null
Long blng = null
if(d['K']){
glat = d['L'].toString().toDouble() * 10000000000L
glng = d['K'].toString().toDouble() * 10000000000L
def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3)
blng = baidu.x * 10000000000L
blat = baidu.y * 10000000000L
}
def estateType2 = "普通住宅"
if(name.contains("别墅")){
estateType2 = "别墅"
}else if(name.contains("商住")){
estateType2 = "商住"
}else{
def type = d['I']
if(type){
if(type.contains("别墅")){
estateType2 = "别墅"
}else if(type.contains("商住")){
estateType2 = "商住"
}
}
}
def key = city+"##"+newRegion+"##"+name
if(name && !keys.contains(key)){
def dist = CommonDatautils.findDistrict(city, newRegion, name, "district")
if(!dist){
keys << key
def data = new BasicDBObject();
data.put("name_cat",0)
data.put("status",0)
data.put("cat","district")
data.put("c_date", new Date())
data.put("m_date", new Date())
data.put("city", city)
data.put("region",newRegion)
data.put("name", name)
data.put("estate_type2",estateType2)
data.put("address", formatAddress)
if(glat){
data.put("lat",glat)
data.put("lng",glng)
data.put("lat2",blat)
data.put("lng2",blng)
}
def markMap = [:]
markMap.put("data_source", "机器入库")
data.put("mark_map",markMap)
index++
DbUtils.seaweed.save(data);
println index+" "+data
addPrice(data)
}
}
}else{
println city +" "+region +" 区域无法标准化"
}
}
},0)
}
def addPrice(dist){
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-03-01")
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-25")
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
for(def weekend : weekends){
CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0)
}
}
def nameFileter(def name){
name = name.replace("·","").replace(".","").replace("|","")
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
if(name.contains("(")){
def delstr = name.substring(name.indexOf("("), name.indexOf(")")+1)
name = name.replace(delstr,"")
}
return name;
}
def findAvgPrice(def commid){
def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2";
def avg_price = null
sqlInstance.eachRow(sql, { r ->
avg_price = r.target_avg_price
});
return avg_price
}
}
@@ -0,0 +1,280 @@
package com.fangjia.script.chongqing
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.gj.paimai.DataUtil
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import groovy.sql.Sql
import java.util.concurrent.CopyOnWriteArrayList
import com.fangjia.script.gj.district.FangjiaClientUtils
class MatchEvaluateTest {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def path = "D:\\数据\\重庆农商行\\2022-05-06\\模拟_测试数据.xlsx"
def total = 1500
static main(args) {
def task = new MatchEvaluateTest()
// task.run()//小区匹配
// task.run1()//地图匹配
//// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除
// task.addPrice()
// task.addArroundPrice()
// task.addAuthor()
task.stat()
}
def addArroundPrice(){
ExcelUtils.multiThreadingModify(path, { row ->
if(row['A'] && !row['K']) {
def arroundObj = FangjiaClientUtils.arroundPrice(row['A'], row['B'])
if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
def priceObj = arroundObj.result
row['K'] = priceObj.unitPrice
row['L'] = (int) (priceObj.unitPrice * row['C'])
row['O'] = "周边价格"
}
}
});
}
def run(){
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['D'] && row['B']){
def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['D'] = jsonObject.get("id")
row['E'] = jsonObject.get("city")
row['F'] = jsonObject.get("district")
row['G'] = jsonObject.get("name")
row['H'] = jsonObject.get("address")
row['I'] = jsonObject.get("propertyType")
row['J'] = jsonObject.get("map")
// row['K'] = jsonObject.get("lng")
// row['L'] = jsonObject.get("lat")
}
}
});
}
def run1(){
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['D'] && row['B']){
def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], null, true, null, null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['D'] = jsonObject.get("id")
row['E'] = jsonObject.get("city")
row['F'] = jsonObject.get("district")
row['G'] = jsonObject.get("name")
row['H'] = jsonObject.get("address")
row['I'] = jsonObject.get("propertyType")
row['J'] = jsonObject.get("map")
// row['K'] = jsonObject.get("lng")
// row['L'] = jsonObject.get("lat")
}
}
});
}
def addPrice(){
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['J'] && row['E']){
def priceObj = CommonDatautils.findDistrictPrice(row['E'], row['F'], row['G'])
if(priceObj){
row['K'] = priceObj.avg_price
row['L']= (int)(priceObj.avg_price * row['C'])
}
def avg_price = findAvgPrice(row['D'])
if(row['K'] && avg_price){
def weekendPrice = row['K'].toString().toDouble()
def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
row['M'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15)
}
}
});
}
def addAuthor(){
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
def authors = ["丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"]
def personTotal = authors.size()
def avg = Math.ceil(total/personTotal)
def index = 0
def count = 0;
ExcelUtils.modify(path, { row ->
if(row['B']){
if(index < avg){
row['N'] = authors[count]
}
index++
if(index >= avg){
count++
index = 0
}
}
});
}
def addDistrict(){
def path = "D:\\数据\\重庆农商行\\2022-05-06\\需新增小区.xlsx"
def keys = new CopyOnWriteArrayList()
def index = 0
ExcelUtils.modify(path, { d->
def flag = d['M']
if(!flag){
def city = d['E']
def region = d['F']
def newRegion = RegionData.formatRegion(city, region)
if(newRegion){
def name = d['G']
name = nameFileter(name)
def formatAddress = null
def address = d['H']
if(address){
formatAddress = AddressUtil.formatAddress(city, address)
if(!formatAddress){
if(address.contains("交叉口")){
formatAddress = address.substring(0, address.toString().indexOf("交叉口")+3)
}else if(address.contains("交叉")){
formatAddress = address.substring(0, address.toString().indexOf("交叉")+2)
}
}
}
Long glat = null
Long glng = null
Long blat = null
Long blng = null
if(d['K']){
glat = d['L'].toString().toDouble() * 10000000000L
glng = d['K'].toString().toDouble() * 10000000000L
def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3)
blng = baidu.x * 10000000000L
blat = baidu.y * 10000000000L
}
def estateType2 = "普通住宅"
if(name.contains("别墅")){
estateType2 = "别墅"
}else if(name.contains("商住")){
estateType2 = "商住"
}else{
def type = d['I']
if(type){
if(type.contains("别墅")){
estateType2 = "别墅"
}else if(type.contains("商住")){
estateType2 = "商住"
}
}
}
def key = city+"##"+newRegion+"##"+name
if(name && !keys.contains(key)){
def dist = CommonDatautils.findDistrict(city, newRegion, name, "district")
if(!dist){
keys << key
def data = new BasicDBObject();
data.put("name_cat",0)
data.put("status",0)
data.put("cat","district")
data.put("c_date", new Date())
data.put("m_date", new Date())
data.put("city", city)
data.put("region",newRegion)
data.put("name", name)
data.put("estate_type2",estateType2)
data.put("address", formatAddress)
if(glat){
data.put("lat",glat)
data.put("lng",glng)
data.put("lat2",blat)
data.put("lng2",blng)
}
def markMap = [:]
markMap.put("data_source", "机器入库")
data.put("mark_map",markMap)
index++
DbUtils.seaweed.save(data);
println index+" "+data
addPrice(data)
}
}
}else{
println city +" "+region +" 区域无法标准化"
}
}
},0)
}
def addPrice(dist){
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-03-01")
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-25")
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
for(def weekend : weekends){
CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0)
}
}
def nameFileter(def name){
name = name.replace("·","").replace(".","").replace("|","")
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
if(name.contains("(")){
def delstr = name.substring(name.indexOf("("), name.indexOf(")")+1)
name = name.replace(delstr,"")
}
return name;
}
def findAvgPrice(def commid){
def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2";
def avg_price = null
sqlInstance.eachRow(sql, { r ->
avg_price = r.target_avg_price
});
return avg_price
}
def stat(){
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\2022-05-06\\模拟_测试数据_合并后 (1).xlsx",0,1)
ExcelUtils.multiThreadingModify(path, { d ->
if(d['B']){
for(def row : datas){
if(d['B']==row['B']){
if(row['K']){
d['P'] = row['K']
break
}
}
}
}
});
}
}
@@ -0,0 +1,510 @@
package com.fangjia.script.chongqing
import com.fangjia.core.util.SourceNameUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import java.util.concurrent.CopyOnWriteArrayList
class PocTest {
static main(args) {
def task = new PocTest();
task.run();
}
def run(){
// match("D:\\数据\\重庆农商行\\poc\\test.xlsx")
// addPrice("D:\\数据\\重庆农商行\\poc\\test.xlsx")
// standard("D:\\数据\\重庆农商行\\poc\\test.xlsx")
// addLoudongNum("D:\\数据\\重庆农商行\\poc\\test.xlsx")
// addDetail("D:\\数据\\重庆农商行\\poc\\test.xlsx", "D:\\数据\\重庆农商行\\poc\\楼盘详情.xlsx")
// createLoudong("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\底层数据-楼栋-户.xlsx")
// createPaimai("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\法拍案例.xlsx")
// createHouse("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\案例.xlsx")
// createDealPrice("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\案例.xlsx")
createEvaluation("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\案例.xlsx")
}
def match(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['P'] && row['C']){
def jsonObject = OpenApiClient.addressMatch(row['A'], row['C'], "property", false, row['B'], null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['P'] = jsonObject.get("name")
row['Q'] = jsonObject.get("id")
}
}
})
}
def addPrice(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(row['Q']){
def dist = CommonDatautils.findDistrictById(row['Q'])
if(dist){
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
if(priceObj){
row['E'] = priceObj.avg_price
}
}
}
})
}
def standard(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(row['C']){
def jsonObject = OpenApiClient.standard2Action(row['A'], row['C'])
if(jsonObject){
println jsonObject
row['F'] = jsonObject.get("province")
row['G'] = jsonObject.get("city")
row['H'] = jsonObject.get("district")
row['I'] = jsonObject.get("township")
row['J'] = jsonObject.get("street")
row['K'] = jsonObject.get("streetNumber")
row['L'] = jsonObject.get("buildingNumber")
row['M'] = jsonObject.get("unitNumber")
row['N'] = jsonObject.get("floor")
row['O'] = jsonObject.get("roomNumber")
}
}
})
}
def addLoudongNum(path){
ExcelUtils.multiThreadingModify(path, { row ->
if (row['Q'] && row['L']) {
def jsonObject = OpenApiClient.searchLoudong(row['Q'], row['L'], row['M'])
if (jsonObject && jsonObject.result) {
if(jsonObject.result.size()>=1){
def obj = jsonObject.result[0]
row['S'] = obj.id
row['R'] = obj.houseNumber
}
}
if(row['O']){
def jsonObject1 = OpenApiClient.searchHouseLoudong(row['Q'], row['L'], row['M'], row['O'])
if (jsonObject1 && jsonObject1.result) {
if(jsonObject.result.size()>=1){
def obj = jsonObject.result[0]
row['U'] = obj.id
row['T'] = obj.roomNum
}
}
}
}
})
}
def createLoudong(path, output){
def rets = new CopyOnWriteArrayList()
def datas = ExcelUtils.read(path, 0, 1)
rets << ["输入城市","输入区域","输入地址","面积","楼栋号","楼栋ID","物业类型","主朝向","地上层数","地下层数","竣工日期","梯","户","总户数","景观","户号","户ID","主朝向","景观","户型结构"]
for(def row : datas){
def data = []
data << row['A']
data << row['B']
data << row['C']
data << row['D']
if (row['Q'] && row['L']) {
def jsonObject = OpenApiClient.searchLoudong(row['Q'], row['L'], row['M'])
if (jsonObject && jsonObject.result && jsonObject.result.size()>=1) {
def obj = jsonObject.result[0]
data << obj.houseNumber
data << obj.id
data << obj.type// 物业类型
data << ""//主朝向
data << obj.floors//地上层数
data << ""//地下层数
data << obj.buildYear;//竣工日期
data << ""//梯
data << ""//户
data << obj.households//总户数
data << ""//景观
}else{
data << ""
data << ""
data << ""// 物业类型
data << ""//主朝向
data << ""//地上层数
data << ""//地下层数
data << ""//竣工日期
data << ""//梯
data << ""//户
data << ""//总户数
data << ""//景观
}
if(row['O']){
def jsonObject1 = OpenApiClient.searchHouseLoudong(row['Q'], row['L'], row['M'], row['O'])
if (jsonObject1 && jsonObject1.result) {
if(jsonObject1.result.size()>=1){
def obj = jsonObject1.result[0]
data << obj.roomNum//户号
data << obj.id//户ID
data << obj.direction//主
data << obj.sight//景观
if(obj.roomNum && obj.roomNum.contains("跃")){
data << "跃式"//户型结构
}else{
data << ""//户型结构
}
}
}
}
}
rets << data
}
ExcelUtils.write(output, rets, "底层数据-楼栋-户")
}
def addDetail(path, output){
def datas = ExcelUtils.read(path, 0, 1)
def rows = []
rows << ["输入城市","输入区域","输入地址","小区名称","小区ID","小区别名","小区地址","板块","街道","基准价","经纬度","竣工时间","开发商名称","绿化率","容积率","车位个数","物业公司","物业费","住宅幢数","总户数","车位配比","占地面积","是否学区","周边教育","周边交通","周边商业","周边医院","周边金融机构"]
for(def d : datas){
if(d['Q']){
def dist = CommonDatautils.findDistrictById(d['Q'])
if(dist){
def row = []
def jsonObject = OpenApiClient.districtDetail(dist.city, dist.region, dist.name)
if(jsonObject) {
row << d['A']
row << d['B']
row << d['C']
row << jsonObject.get("name")//小区名称
row << d['Q']//小区ID
if(jsonObject.get("alias")){
row << jsonObject.get("alias").join(",").replace('"',"")//小区别名
}else{
row << ""
}
row << jsonObject.get("address")//小区地址
row << jsonObject.get("commercialArea")//板块
row << jsonObject.get("township")//街道***********
row << jsonObject.get("avgPrice")//基准价
row << jsonObject.get("lng").toString()+","+jsonObject.get("lat").toString()//经纬度
row << jsonObject.get("finishBuildTime")//竣工时间
row << jsonObject.get("developer")//开发商名称
row << SeaweedUtils.formatVirescencePercent(jsonObject.get("greenPercentage"))//绿化率
row << SeaweedUtils.formatCubagePercent(jsonObject.get("plotRatio"))//容积率
row << SeaweedUtils.formatCarSiteCount(jsonObject.get("parking"))//车位个数
row << jsonObject.get("propertyManagement")//物业公司
row << jsonObject.get("managementFee")//物业费
row << jsonObject.get("buildingCount")//住宅幢数
row << SeaweedUtils.formatHouseholdCount(jsonObject.get("totalUnitCount"))//总户数
row << SeaweedUtils.calcCarAndHouseholdRadio(jsonObject.get("parking"), jsonObject.get("totalUnitCount"))//车位配比*********
row << SeaweedUtils.formatAllArea(jsonObject.get("totalUnitCount"))//占地面积
}
def facilityObj = OpenApiClient.facilitys(dist.city, dist.region, dist.name)
if(facilityObj && facilityObj.result){
def result = facilityObj.result
if(result.keySchool){
row << '是'
}else {
row << ""
}
if(result.school){
row << result.school.join(",").replace('"',"")
}else {
row << ""
}
def bus = []
if(result.bus){
bus.addAll(result.bus)
}
if(result.metro){
bus.addAll(result.metro)
}
if(bus){
row << bus.join(",").replace('"',"")
}else{
row << ""
}
if(result.supermarket){
row << result.supermarket.join(",").replace('"',"")
}else{
row << ""
}
if(result.hospital){
row << result.hospital.join(",").replace('"',"")
}else{
row << ""
}
if(result.bank){
row << result.bank.join(",").replace('"',"")
}else{
row << ""
}
}
rows << row
}
}
}
ExcelUtils.write(output, rows, "底层数据-小区")
}
def createPaimai(path, output){
def datas = ExcelUtils.read(path, 0, 1)
def rows = []
rows << ["输入城市","输入区域","输入地址","小区名称","标的名称","楼栋","户号","面积","案例时间","类型","状态","成交单价","评估单价"]
def keys = []
for(def d : datas){
if(d['Q']) {
def dist = CommonDatautils.findDistrictById(d['Q'])
if (dist) {
def key = dist.city+"#"+dist.region+"#"+dist.name
if(keys.contains(key)){
continue
}
keys << key
def jsonObject = OpenApiClient.searchPaimais(dist.city, dist.region, dist.name,10)
if(jsonObject && jsonObject.result){
def results = jsonObject.result
for(def r: results){
def row = []
row << d['A']
row << d['B']
row << d['C']
row << r.matchName
row << r.address
row << r.houseNum
row << r.roomNum
row << r.area
row << r.startDate
row << r.count
row << r.status
row << r.tradeAvgPrice
row << r.tradePrice
row << r.id
rows << row
}
println dist.name +" "+results.size()
}
}
}
}
ExcelUtils.write(output, rows, "拍卖案例")
}
def createHouse(path, output){
def datas = ExcelUtils.read(path, 0, 1)
def rows = []
rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
def keys = []
for(def d : datas) {
if (d['Q']) {
def key = d['Q']
if (keys.contains(key)) {
continue
}
keys << key
def jsonObject = OpenApiClient.houseList(d['Q'],"20210501",20)
if(jsonObject && jsonObject.result) {
def results = jsonObject.result
for (def r : results) {
def row = []
row << "挂牌"
row << r.districtName
if(r.estateType){
row << r.estateType
}else{
if(r.purpose){
row << r.purpose
}else{
row << ""
}
}
if(r.room){
def roomStr = r.room+"室"
if(r.hall){
roomStr = roomStr +r.hall+"厅"
}
row << roomStr
}else{
row << ""
}
row << r.area
row << r.fitment
row << r.pDate
row << ""
row << r.avgPrice
row << r.totalPrice
if(r.floor){
row << r.floor
}else{
row << ""
}
if(r.height){
row << r.height
}else{
row << ""
}
row << r.direction
row << SourceNameUtil.getResourceName(r.root)
rows << row
}
println d['Q'] +" "+results.size()
}
}
}
ExcelUtils.write(output, rows, "挂牌案例")
}
def createEvaluation(path, output){
def datas = ExcelUtils.read(path, 0, 1)
def rows = []
rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
def keys = []
for(def d : datas) {
if (d['Q']) {
def key = d['Q']
if (keys.contains(key)) {
continue
}
keys << key
def jsonObject = OpenApiClient.evaluaionList(d['Q'],"20210501",20)
if(jsonObject && jsonObject.result) {
def results = jsonObject.result
for (def r : results) {
def row = []
row << "评估"
row << r.commName
row << ""
if(r.roomcnt){
def roomStr = r.roomcnt+"室"
if(r.livingroomcnt){
roomStr = roomStr +r.livingroomcnt+"厅"
}
row << roomStr
}else{
row << ""
}
row << r.buildArea
row << r.fitment
row << r.cDate.substring(0, r.cDate.indexOf("T"))
row << ""
row << r.unitPrice
row << r.totalPrice
if(r.floor){
row << r.floor
}else{
row << ""
}
if(r.floorcnt){
row << r.floorcnt
}else{
row << ""
}
if(r.houseaspect && !r.houseaspect.contains("暂")){
row << r.houseaspect
}else{
row << ""
}
row << "房价网"
rows << row
}
println d['Q'] +" "+results.size()
}
}
}
ExcelUtils.write(output, rows, "评估案例")
}
def createDealPrice(path, output){
def datas = ExcelUtils.read(path, 0, 1)
def rows = []
rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
def keys = []
for(def d : datas) {
if (d['Q']) {
def key = d['Q']
if (keys.contains(key)) {
continue
}
keys << key
def jsonObject = OpenApiClient.searchDealPrice(d['Q'],"20210501",20)
if(jsonObject && jsonObject.result) {
def results = jsonObject.result
for (def r : results) {
def row = []
row << "成交"
row << r.districtName
if(r.type){
row << r.type
}else{
row << ""
}
if(r.room){
def roomStr = r.room+"室"
if(r.hall){
roomStr = roomStr +r.hall+"厅"
}
row << roomStr
}else{
row << ""
}
row << r.area
row << r.fitment
row << r.tradeDate.substring(0, r.tradeDate.indexOf("T"))
row << ""
row << r.avgPrice
row << r.totalPrice
if(r.floor){
row << r.floor
}else{
row << ""
}
if(r.height){
row << r.height
}else{
row << ""
}
if(r.direction && !r.direction.contains("暂")){
row << r.direction
}else{
row << ""
}
row << r.source
rows << row
}
println d['Q'] +" "+results.size()
}
}
}
ExcelUtils.write(output, rows, "成交案例")
}
}
@@ -830,6 +830,7 @@ class CommonDatautils {
keys.put("name_cat", 1);
keys.put("finish_building_date", 1)
keys.put("mapping_alias", 1)
keys.put("sid", 1)
DSCursor cursor = getDataStoreConnection().find(q, keys);
if(cursor){
for(def obj : cursor){
@@ -3166,5 +3167,22 @@ class CommonDatautils {
}
return avg_price;
}
static def findCityAvgRentRadio(def city, def weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", weekend)
def dbcursor = SQMETRE_RENT_COLL.find(q)
def avg_price = null
def radios = []
for(def d : dbcursor){
if(d.price_to_rent){
radios << d.price_to_rent.toInteger()
}
}
def avg_radios = (int)(radios.sum()/radios.size())
return avg_radios;
}
}
@@ -16,9 +16,7 @@ import org.apache.commons.lang.math.RandomUtils;
class DataHandleUtils {
static main(args){
}
static def getStandardRegion(def regionMap, def region){
def newRegion = null;
@@ -94,6 +92,8 @@ class DataHandleUtils {
BigDecimal deSource = new BigDecimal(dSource);
return deSource.setScale(scale, BigDecimal.ROUND_HALF_UP).doubleValue();
}
static def trimChar(String c) {
try {
@@ -35,7 +35,7 @@ class DistrictAroundFacilitiesBatch extends BaseBatch {
@Override
public void process(Object args) {
def cities = ["北京", "上海"]
def cities = ["北京", "上海"]//
for (def city : cities) {
def ret = CommonDatautils.getAllDistricts("district", city)
// def ret = getAllDistrictBaseInfo("上海", '闵行', '万源城', "district");
@@ -42,17 +42,17 @@ class YouDaDataParser {
static def new_queryUrl = "http://192.168.0.228:9200/new_deal_price_lansi/doc/_search"
static def esf_targetUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/doc"
static def esf_targetUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/_doc"
static def esf_queryUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/doc/_search"
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2022-02-01"
static def statDate = "2022-03-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
// task.esfParser();//友达二手房存入mongo
task.esfParser();//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
@@ -285,7 +285,7 @@ class YouDaDataParser {
def esfParser(){
def files = ["D:\\数据\\友达\\2022年3月\\查询1.DBF"]
def files = ["D:\\数据\\友达\\2022年4月\\查询1.DBF"]
def datas = []
for(def fileName : files){
def rows = readDBF(fileName);
@@ -0,0 +1,61 @@
package com.fangjia.script.gj.apiLog
import com.alibaba.excel.util.DateUtils
import com.fangjia.core.util.DateUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.Calendar;
@BatchOwner(Watcher.GaoJuan)
class ApiTimeOutAnalyBatch extends BaseBatch{
DBCollection LOG = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.42", 52002)
static main(args) {
runBatch(new ApiTimeOutAnalyBatch(), args);
}
@Override
public void process(Object args) {
def uriCountMap = findTimeOutUri()
sendMessage(uriCountMap)
}
def sendMessage(uriCountMap){
def message = "【Api接口近5分钟超过5条请求时间超3秒提醒】\n超时接口:"
def total = message.length()
uriCountMap.each{uri,count->
if(count>=5){
message = message + uri +"\n"
}
}
if(message.length()>total){
println message
def access_token = "1e25f620ea2be926b2441e4f30d67f37abdff2e14088b03b41b4082b11fc34ea"
DingDingMessageUtil.sendTextMessage(access_token, message)
}
}
def findTimeOutUri(){
def q = new BasicDBObject()
q.put("time", new BasicDBObject('$gte', 3000))
def startTime = DateUtil.add(new Date(), Calendar.MINUTE, -5)
q.put("c_date", new BasicDBObject('$gte', startTime))
def dbcursor = LOG.find(q)
def uriCountMap = [:]
for(def d : dbcursor){
def count = uriCountMap.get(d.request_uri)
if(!count){
count = 0
}
uriCountMap.put(d.request_uri, count + 1)
}
return uriCountMap
}
}
@@ -0,0 +1,111 @@
package com.fangjia.script.gj.apiLog
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
@BatchOwner(Watcher.GaoJuan)
class ApiUserCountAnalyBatch extends BaseBatch{
static final dbFangjia = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia?autoReconnect=true&characterEncoding=utf-8&autoReconnect=true", "sa", "G00jia", "com.mysql.jdbc.Driver");
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection AUTHORITY_COLL = MongoUtils.getCollectionByName2("fangjia", "authority.api_user_authority", lineUrl)
DBCollection VISIT_COLL = MongoUtils.getCollectionByName2("business", "api_visit_count", lineUrl)
static main(args) {
runBatch(new ApiUserCountAnalyBatch(), args);
}
@Override
public void process(Object args) {
def rets = findAllUsers()
sendMessage(rets)
}
def sendMessage(def rets){
def message = "【Api接口使用提醒】\n"
def total = message.length()
for(def r : rets){
def type = "正式帐户"
if(r.type!=0){
type = "测试帐户"
}
message = message + r.company+ " 用户ID:"+r.user_id+"(${type}),有 "+r.uris.size()+" 个接口使用率已超过80%,接口有:"+r.uris.join(",")+"\n"
}
if(message.length()>total){
println message
def access_token = "1e25f620ea2be926b2441e4f30d67f37abdff2e14088b03b41b4082b11fc34ea"
DingDingMessageUtil.sendTextMessage(access_token, message)
}
}
def findAllUsers(){
def sql = "select id,COMPANY,is_test from api_user where dead_time>='"+DateUtils.formatDateByPattern(new Date(),'yyyy-MM-dd')+"'"
def rows = dbFangjia.rows(sql)
def rets = []
for(def row : rows){
def obj = findResources(row.id)
if(obj){
def map = obj.total_count_map
def uris = []
map.each{k,v->
//def visit_count = findVisitCount(row.id, k)
def total_visit_count = findAllVisitCount(row.id, k)
println row.id +" "+k+" "+v+" "+total_visit_count
if(v && total_visit_count){
def precent = v*0.8;
if(total_visit_count>=precent){
uris << k
}
}
}
if(uris){
rets << ["user_id":row.id,"company":row.company,"uris":uris,"type":row.is_test]
}
}
}
return rets;
}
def findAllVisitCount(userId, uri){
def q = new BasicDBObject()
q.put("user_id", userId.toString())
q.put("uri", uri)
q.put("visit_date", new BasicDBObject('$lte',DateUtils.truncateDay(new Date())))
def dbcursor = VISIT_COLL.find(q)
def total = 0
for(def obj : dbcursor){
total = total +obj.visit_count
}
return total
}
def findVisitCount(userId, uri){
def q = new BasicDBObject()
q.put("user_id", userId.toString())
q.put("uri", uri)
q.put("visit_date", DateUtils.truncateDay(new Date()))
def obj = VISIT_COLL.findOne(q)
if(obj){
return obj.visit_count
}
return null
}
def findResources(def userId){
def q = new BasicDBObject()
q.put("user_id", userId)
return AUTHORITY_COLL.findOne(q)
}
}
@@ -50,4 +50,32 @@ class FileUtils {
}
out.close();
}
static def createTxtFile(ret, fileName, charset, isAppend){
File file = new File(fileName);
if (!file.getParentFile().exists()) {
boolean result = file.getParentFile().mkdirs();
if (!result) {
System.out.println("´´½¨Ä¿Â¼Ê§°Ü");
}
}
OutputStream out = new FileOutputStream(fileName, isAppend);
def sb = new StringBuffer();
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
def index = 0
for(def row : ret){
if(row){
index++
sb.append(row)
sb.append("\n");
}
}
if(sb.length()>0){
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
}
out.close();
}
}
@@ -25,11 +25,11 @@ class DataMatch {
static main(args) {
def task = new DataMatch()
// task.run1()
task.run2()
// task.run2()
// task.run3()
// println task.findAddress("²ıÖݽֵÀÌÄÏã·")
// task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx")
task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx")
}
def run1(){
@@ -0,0 +1,246 @@
package com.fangjia.script.gj.crawl
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.jsoup.Jsoup
import org.openqa.selenium.By
import org.openqa.selenium.WebDriver
import org.openqa.selenium.WebElement
import org.openqa.selenium.chrome.ChromeDriver
class AnjukeDetailCrawl {
DBCollection detailColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_detail")
static main(args) {
def task = new AnjukeDetailCrawl();
task.run();
}
def run(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
def url = "https://chongqing.anjuke.com/community/?from=navigation"
WebDriver driver = new ChromeDriver()
def regions = getRegions(driver, url)
getAllRegions(driver, regions)
}
def getAllRegions(driver, regions){
for(def url : regions){
driver.get(url)
def doc = Jsoup.parse(driver.getPageSource());
def total = getTotal(doc)
if(total) {
if (total > 1250) {
} else {
def links = getAllLinks(url, driver)
parse(links,driver)
}
}
}
}
def getAllLinks(url, driver){
def stop = false
def urls = []
while(!stop) {
def doc = Jsoup.parse(driver.getPageSource());
def details = JQueryUtils.select(".list-cell a.li-row", doc)
for (def d : details) {
urls << d.attr("href")
}
def nextEle = JQueryUtils.select(".next", doc)
if (nextEle) {
driver.findElement(By.cssSelector(".next")).click()
}else{
stop = true
}
if(urls.size()>20){
stop = true
}
}
return urls
}
def getTotal(doc){
def totalText = JQueryUtils.selectText(".total-info", doc)
if(totalText){
def total = totalText.substring(totalText.indexOf("到")+1, totalText.indexOf("个")).trim()
if(total && total.isNumber()){
return total.toInteger()
}
}
return null;
}
def getRegions(driver, url){
driver.get(url)
Thread.sleep(3000)
def doc = Jsoup.parse(driver.getPageSource());
// def loginIfm = JQueryUtils.select("#iframeLoginIfm", doc)
// if(loginIfm){
// def href = loginIfm[0].attr("src")
// driver.get(href)
// driver.findElementById("pwdTab").click()
// WebElement elemUsername = driver.findElement(By.id("pwdUserNameIpt")); //获取用户名框
// WebElement elemPassword = driver.findElement(By.id("pwdIpt")); //获取密码框
// WebElement btn = driver.findElement(By.cssSelector("input[class='submit-btn']")); //获取点击登陆按钮。
// elemUsername.sendKeys("17740818975");
// elemPassword.sendKeys("12qwaszx");
// btn.submit(); // 点击行为;
//
// def regionEles = JQueryUtils.select(".region-parents li a", doc)
// def urls = []
// for(def e : regionEles){
// urls << e.attr("href")
// }
// return urls;
// }
def regionEles = JQueryUtils.select(".region-parents li a", doc)
def urls = []
for(def e : regionEles){
urls << e.attr("href")
}
return urls;
}
def parse(def links, driver){
def rets = []
for(def link : links){
def district = parsing(link, driver)
if(district){
rets << district
if (rets.size() % 100 == 0) {
def add= saveData(rets)
println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
rets = []
}
}
}
if (rets.size()> 0) {
def add= saveData(rets)
println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
}
}
def saveData(def results){
def add = 0
for(def ret : results){
def q = new BasicDBObject()
q.put("url", ret.url)
def dbo = detailColl.findOne(q);
DBObject obj = new BasicDBObject();
ret.each{k,v->
obj.put(k,v)
}
if(dbo){
obj.m_date = new Date()
detailColl.update(q, new BasicDBObject('$set', obj))
}else{
add++
obj.c_date = new Date()
obj.m_date = new Date()
try{
detailColl.save(obj)
}catch(Exception e){
println obj.url
}
}
}
return add
}
def parsing(url, driver){
def district = [:]
def stop = false
def count = 0
while(!stop){
driver.get(url)
def doc = Jsoup.parse(driver.getPageSource());
def headers = JQueryUtils.select(".crumbs a", doc)
if(headers){
if (headers && headers.size() >= 3) {
district.city = headers[1].text().replace("小区", "")
district.region = headers[2].text().replace("小区", "")
district.name = headers[headers.size() - 1].text().replace("二手房", "")
}
district.address = JQueryUtils.selectText("div.column-1:contains(小区地址) .value", doc)
def priceStr = JQueryUtils.selectText(".average", doc)
if (priceStr && priceStr.isNumber()) {
district.avg_price = priceStr.toInteger()
}
def infoDiv = JQueryUtils.select(".info-list", doc)
district.estate_type = getValue(infoDiv.select(".column-2:contains(物业类型) .value").text())
district.estate_charge = getValue(infoDiv.select(".column-2:contains(物业费) .value").text())
district.estate_company = getValue(infoDiv.select(".column-1:contains(物业公司) .value").text())
district.developer = getValue(infoDiv.select(".column-1:contains(开发商) .value").text())
def houseCountStr = infoDiv.select(".column-2:contains(总户数) .value").text()
if (houseCountStr) {
houseCountStr = houseCountStr.replace("户", "")
if (houseCountStr) {
district.house_count = houseCountStr.toInteger()
}
}
district.finish_date = getValue(infoDiv.select(".column-2:contains(竣工时间) .value").text())
district.build_area = getValue(infoDiv.select(".column-2:contains(总建面积) .value").text())
def carSiteStr = getValue(infoDiv.select(".column-2:contains(停车位) .value").text())
if(carSiteStr){
if(carSiteStr.contains("(")){
carSiteStr = carSiteStr.substring(0, carSiteStr.indexOf("("))
}
if(carSiteStr && carSiteStr.isNumber()){
district.car_site = carSiteStr
}
}
district.volumetric_rate = getValue(infoDiv.select(".column-2:contains(容积率) .value").text())
district.greening_rate = getValue(infoDiv.select(".column-2:contains(绿化率:) .value").text())
district.build_type = getValue(infoDiv.select(".column-2:contains(建筑类型) .value").text())
district.property_life = getValue(infoDiv.select(".column-2:contains(产权年限) .value").text())
district.car_site_charge = getValue(infoDiv.select(".column-1:contains(停车费) .value").text())
district.url = url
district.c_date = new Date()
district.m_date = new Date()
district.root = "anjuke.com"
stop = true;
}else{
Thread.sleep(3000)
count++
}
if(count>10){
stop = true;
}
}
return district
}
String getValue(String s){
if(s && s.contains("暂无")){
return null
}
return s
}
}
@@ -0,0 +1,29 @@
package com.fangjia.script.gj.crawl
import org.jsoup.Jsoup
class CommonCrawl {
static def getDocument(url, driver){
def stop = false
def error = 0
def doc = null
while(!stop){
try{
driver.get(url)
doc = Jsoup.parse(driver.getPageSource());
if(doc){
stop = true;
}else{
error++
Thread.sleep(5000)
}
}catch(Exception e){
error++
}
if(error>=10){
stop = true;
}
}
return doc
}
}
@@ -17,7 +17,7 @@ class TempMatchDistrict {
}
def stat(){
def citys = ["哈尔滨","齐齐哈尔","大庆","牡丹江","佳木斯","绥化","鸡西","七台河","双鸭山","鹤岗","黑河","伊春","大兴安岭"]
def citys = ["广州","深圳","佛山","东莞","中山","湛江","惠州","江门","珠海","汕头","揭阳","茂名","清远","肇庆","潮州","河源","韶关","阳江","梅州","云浮","汕尾"]
def total = 0
for(def city : citys){
def count = CommonDatautils.findAllDistrictCount("district", city)
@@ -83,6 +83,23 @@ class FangjiaClientUtils {
return jsonObj;
}
public static def arroundPrice(city,address){
def jsonObj = null
def content = null
try {
address = address.replace("{","")
// def url = "http://192.168.0.19:39080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8")
def url = "http://192.168.0.100:38080/FangJiaDataService/evaluate?address="+URLEncoder.encode(address,"UTF-8")+"&city="+URLEncoder.encode(city,"UTF-8")+"&maxNum=10&radius=1000&size=100"
// def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8")
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
public static Map<String, Object> JsonToMap(JSONObject json) {
Map<String, Object> columnValMap = new HashMap<String, Object>();
@@ -58,6 +58,8 @@ class DistrictIndexBatch extends BaseBatch{
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static def month = DateUtils.formatDateByPattern(new Date(),"yyyy-MM")
static main(args) {
runBatch(new DistrictIndexBatch(), args);
}
@@ -71,7 +73,7 @@ class DistrictIndexBatch extends BaseBatch{
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def citys = CommonDatautils.findAllCity();
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2"])
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2","household_count"])
def code = cityCodeDatas.get(city)
def city_avg_total_price = null
if(city!="ÉϺ£"){
@@ -84,7 +86,7 @@ class DistrictIndexBatch extends BaseBatch{
city_avg_total_price = 531.2615913
}
if(city_avg_total_price){
calcute(city, dists, code, city_avg_total_price)
// calcute(city, dists, code, city_avg_total_price)
calcuteIndex(city, dists, code, city_avg_total_price)
}
@@ -131,7 +133,7 @@ class DistrictIndexBatch extends BaseBatch{
map.total_price = total_price
//map.comp_value = potentiality
//map.avg_total_price = total_price_index
map.month = "2022-03"
map.month = month
map.c_date = DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
//datas << [d.city, d.region, d.name, total_price_index, avg_charge_index, rank_index, avg_cost_index,potentiality]
@@ -229,7 +231,7 @@ class DistrictIndexBatch extends BaseBatch{
map.comp_value = potentiality
map.avg_total_price = total_price_index
map.month = "2022-03"
map.month = month
map.m_date = DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
map.finish_date = year
@@ -240,6 +242,15 @@ class DistrictIndexBatch extends BaseBatch{
if(rMap.sell_avg_price && sqmetre_rent){
map.price_to_rent = (rMap.sell_avg_price/sqmetre_rent).toInteger()
}
if(d.household_count){
map.household_count = d.household_count
}
if(d.lat){
def lMap = [:]
lMap.lon = d.lng/10000000000.0
lMap.lat = d.lat/10000000000.0
map.location = lMap
}
def newUrl = targetUrl +"_doc/"+map.id
httpSaveData(newUrl, map)
@@ -0,0 +1,354 @@
package com.fangjia.script.gj.meituan
import com.fangjia.core.RegionData
import com.fangjia.core.util.DateUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.gj.paimai.DataUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.GeoUtils
import com.fangjia.script.util.OpenApiClient
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import org.bson.types.ObjectId
class MatchTest {
static main(args) {
def task = new MatchTest()
// task.run()
// task.addNum()
task.dongguang()
// task.meituanDel()
}
def meituanDel(){
def path = "D:\\数据\\美团\\美团需入库小区合并后.xlsx"
def index = 0
ExcelUtils.modify(path, { row ->
def flag = row['O']
if(flag){
def city = row['F']
def region = row['G']
def name = nameFileter(row['H'])
def obj = null
if(city == "东莞"){
obj = CommonDatautils.findDistrict(city, null, name, "district")
}else{
obj = CommonDatautils.findDistrict(city, region, name, "district")
}
if(obj){
index++
println index +" "+ city +' '+ region+" "+name
def udata = new BasicDBObject()
udata.put("name_cat", 11)
udata.put("status", 1)
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", new ObjectId(obj._id.toString()))
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}
}
},0);
}
def dongguang(){
def path = "D:\\数据\\美团\\美团东莞.xlsx"//美团金融小区测试list-匹配.xlsx
ExcelUtils.modify(path, { row ->
def region = null
if(row['B'] && row['B']=="NULL" ){
region = null
}
def name = nameFileter(row['H'])
def address = row['E']
if (name && name.contains("湖畔新家")) {
def obj = CommonDatautils.findDistrict("东莞", "东莞", name, "district")
if(obj){
def json = GaoDeApiUtils.geo(row['F'], address)
if(json && json.count>=1){
def m = json.geocodes[0]
if(m){
def twinshop = getStreetInfo(m.location)
if(twinshop){
def newRegion = RegionData.formatRegion("东莞",twinshop)
if(newRegion){
def dbObj = CommonDatautils.findDistrict("东莞", newRegion, name, "district")
if(dbObj){
println "delete"
DbUtils.seaweed.remove(new BasicDBObject("_id",obj._id))
}else{
println "update"
def udata = new BasicDBObject()
udata.put("region", newRegion)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata))
}
}
}
// println address +" "+twinshop
}
}
}
}
},0)
}
def getStreetInfo(def glocation){
def township = null
if(glocation){
def jsonObject = GaoDeApiUtils.geocode(glocation)
if(jsonObject && jsonObject.regeocode) {
def addressComponent = jsonObject.regeocode.addressComponent
if(addressComponent){
township = addressComponent.township
}
}
}
return township;
}
def addNum(){
def path = "D:\\数据\\美团\\美团金融小区测试list-所有数据.xlsx"//美团金融小区测试list-匹配.xlsx
def index = 0
ExcelUtils.modify(path, { row ->
index++
row['A'] = index
},0)
}
def run() {
def citys = ["潮州","金华","惠州","东莞","苏州","桂林","廊坊","临沧","通辽","雅安"]//
def weekend = DateUtils.getFirstDayOfWeek(new Date());
def cityLevelMap = [:]
def radioMap = [:]
for(def city : citys){
// def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1));
// def avg_radio = CommonDatautils.findCityAvgRentRadio(city, lastMonthDay)
// radioMap.put(city, avg_radio)
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region","name"])
def priceMap = CommonDatautils.queryDistrictPrice(city, weekend)
def datas = addPriceData(dists, priceMap);
datas = calcuteLevel(datas);
cityLevelMap.put(city, ["price":priceMap, "level":datas])
println city +" 楼盘评级同步完成"
}
def path = "D:\\数据\\美团\\美团金融小区测试list-所有数据-评级.xlsx"//美团金融小区测试list-所有数据-评级.xlsx
ExcelUtils.multiThreadingModify(path, { row ->
def region = null
if(row['C'] && row['C']=="NULL" ){
region = null
}
def name = row['D']
def address = row['F']
def city = RegionData.formatCity(row['B'])
if(name && !row['N']){
if(city){
def map = cityLevelMap.get(city)
def rMap = null
if(map){
rMap = getArroundDistrictPriceByAddress(city, address, map.level)
}
if(rMap){
if(rMap.rank){
row['Q'] = rMap.rank
}
if(rMap.level){
row['P'] = rMap.level
}
}
}
}
if (name && row['N']) {
//// def jsonObject = OpenApiClient.addressMatch(row['A'], address, null, true, region, name)
// def jsonObject = OpenApiClient.addressMatch(row['A'], name, null, true, region, null)
// if (jsonObject != null && jsonObject.precise != -1) {
// row['F'] = jsonObject.get("city")
// row['G'] = jsonObject.get("district")
// row['H'] = jsonObject.get("name")
// row['I'] = jsonObject.get("propertyType")
// row['J'] = jsonObject.get("map")
// row['K'] = jsonObject.get("lng")
// row['L'] = jsonObject.get("lat")
// }
if(row['G'] && row['H'] && row['I']){
// def priceObj = CommonDatautils.findDistrictPrice(row['F'], row['G'], row['H'])
// if(priceObj){
// row['M'] = priceObj.avg_price
// }
// def sqmetreRentPrice = CommonDatautils.findNewSqmetreRent(row['F'], row['G'], row['H'])
// if(sqmetreRentPrice){
// row['N'] = sqmetreRentPrice
// }
if(row['N'] && !row['O']){
def avg_rent_radio = radioMap.get(city)
row['O'] = (int)(row['N']/avg_rent_radio)
}
}
}
})
}
def getArroundDistrictPriceByAddress(def city, def address, def levels){
def json = GaoDeApiUtils.geo(city, address)
def location = null
def detail = new BasicDBObject()
def level = null
def rank = null
if(json && json.count>=1){
def m = json.geocodes[0]
if(m){
location = m.location
level = m.level
}
}
if(location){
def arrs = location.split(",")
detail.lng = arrs[0].toString().toDouble()*10000000000L
detail.lat = arrs[1].toString().toDouble()*10000000000L
}
if(detail.lng){
detail.city = city
List<DBObject> list = DbUtils.findAroundDistrictByRadius(detail, 3000)
if(list && list.size()>10){
list = sortByDistance(detail, list);
list = list[0..9]
}
def arroundAvgPrice = getArroundAvgPrice(list)
if(arroundAvgPrice){
def mod = Integer.MAX_VALUE;
for(def d : levels){
def newMod = java.lang.Math.abs( d.avg_price - arroundAvgPrice)
if(newMod<mod){
mod = newMod
rank = d.level
}
}
}
}
def m = [:]
m.level = level
m.rank = rank
return m
}
def sortByDistance(d, list){
for(def nb : list){
def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0)
nb.distance = distance
}
//排序
Collections.sort(list, new Comparator() {
@Override
int compare(Object o1, Object o2) {
if(o1.distance > o2.distance)
return 1;//第二个元素(o1)比第一个元素(o2)大,返回-1
if(o1.distance == o2.distance)
return 0;
return -1;
}
})
return list;
}
def addPriceData(dists, priceMap){
def newLists = []
for(def d : dists){
def price = priceMap.get(d.city+"##"+d.region+"##"+d.name);
if(price){
d.avg_price = price;
newLists << d;
}
}
if(newLists){
newLists.sort {a,b->
def v1 = a.avg_price
def v2 = b.avg_price
return v1.compareTo(v2)
}
}
return newLists;
}
def calcuteLevel(def datas){
def total = datas.size()
def v = (int)(total/10)
def start = 0;
def end = 0;
def map = [:]
for(int i=1;i<=10;i++){
end = end + v;
if(i == 10){
map.put(i, [start, total])
}else{
map.put(i, [start, end])
}
start = end;
}
for(def d=0;d<datas.size();d++){
def dist = datas[d]
for (def o in map) {
def ranges = o.value
def m = ranges[0]
def n = ranges[1]
if(d==0){
if(d<=n){
dist.level = 10 - o.key + 1
break
}
}else{
if(d>m && d<=n){
dist.level = 10 - o.key + 1
break
}
}
}
}
return datas;
}
def getArroundAvgPrice(def list){
def prices = []
for(def l : list){
def priceObj = CommonDatautils.findDistrictPrice(l.city, l.region, l.name)
if(priceObj){
prices << priceObj.avg_price
}
}
if(prices){
def avg_price = (prices.sum()/prices.size()).toInteger()
return avg_price;
}
return null
}
def nameFileter(def name){
name = name.replace("·","").replace(".","").replace("|","")
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
return name;
}
}
@@ -45,9 +45,21 @@ public class BeiKeCrawlByChrome {
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
def cityMap = getAllCityUrls(driver)
def citys = getAllCityUrls(driver)
citys = citys.reverse()
def start = 0
// for(def i=0;i<citys.size();i++){
// def m = citys[i]
// println i+" "+ m.city
// if(m.city=="Ìì½ò"){
// start = i
// break;
// }
//
// }
// citys = citys[(start+1)..(citys.size()-1)]
getLogin(driver)
crawlData(driver, cityMap)
crawlData(driver, citys)
}
@@ -85,13 +97,13 @@ public class BeiKeCrawlByChrome {
}
}
def crawlData(driver, cityMap){
def crawlData(driver, citys){
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
cityMap.each{city, cityUrl->
for(def m : citys){
//def detailUrls = getPageUrls(driver, cityUrl)
def detailUrls = getPageDeail(driver, city, cityUrl)
println cityUrl +" ÏêÇéÁ´½Ó£º"+detailUrls.size()
def detailUrls = getPageDeail(driver, m.city, m.url)
println m.city +" ÏêÇéÁ´½Ó£º"+detailUrls.size()
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
}
}
@@ -141,7 +153,7 @@ public class BeiKeCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
if(dealDate && dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e)
@@ -150,6 +162,10 @@ public class BeiKeCrawlByChrome {
break;
}
}
if(page>=100){
stop = true;
break;
}
page++;
}else{
stop = true;
@@ -182,7 +198,7 @@ public class BeiKeCrawlByChrome {
}
def areaStr = arrs[2].replace("ƽÃ×", "")
if (areaStr) {
if (areaStr && areaStr.isNumber()) {
detail.area = areaStr.toDouble()
}
@@ -337,17 +353,21 @@ public class BeiKeCrawlByChrome {
driver.get("https://www.ke.com/city/")
Thread.sleep(8000)
def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
def cityMap = [:]
def cityEles = JQueryUtils.select(".city_list_ul .city_province a", cityDoc)
def list = []
if(cityEles){
for(def e : cityEles){
def href = e.attr("href")
if(e.text().contains("¸£ÖÝ")){
cityMap.put(e.text(), "https:"+href)
if(e.text().contains("ÖØÇì")){
def cityMap = [:]
cityMap.city = e.text()
cityMap.url = "https:"+href
list << cityMap
//cityMap.put(e.text(), "https:"+href)
}
}
}
return cityMap
return list
}
def getLogin(driver){
@@ -42,7 +42,6 @@ System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Loca
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
def proviceMap = getDocument(driver);
def citys = getCity(driver,proviceMap)
@@ -59,9 +58,11 @@ def getDetailLinks(driver, cityList){
citys.each{city,id->
def urls = []
if(!zxcitys.contains(cityMap.provice)){
driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&city=${id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport=");
def url = "https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&city=${id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport="
getDocument(url, driver)
}else{
driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport=");
def url = "https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport="
getDocument(url, driver)
}
def doc = Jsoup.parse(driver.getPageSource());
@@ -89,7 +90,7 @@ def getDetailLinks(driver, cityList){
}
}
def rets = batchParse(urls);
def rets = batchParse(urls, driver);
saveToDB(rets);
println city +" "+urls.size()
}
@@ -124,7 +125,7 @@ def getCity(driver, proviceMap){
for(def c : cityEles){
def cityid = c.attr("data-cityid")
def city = c.text()
if(!["石家庄市","太原市","长治市","朔州市","廊坊市","衡水市","唐山市","秦皇岛市","邯郸市","邢台市","保定市","沧州市","张家口市","承德市"].contains(city)){
if(!["石家庄市","太原市","齐齐哈尔市","哈尔滨市","鸡西市","长治市","朔州市","廊坊市","衡水市","唐山市","秦皇岛市","邯郸市","邢台市","保定市","沧州市","张家口市","承德市"].contains(city)){
cityMap.put(city, cityid)
}
@@ -149,7 +150,7 @@ def getCity(driver, proviceMap){
def getDocument(driver){
driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=9&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&canLoan=&isRestricted=");
driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=1&canLoan=&isRestricted=&insuranceSupport=");
WebDriverWait wait = new WebDriverWait(driver,20)
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("lots-list"))));
def doc = Jsoup.parse(driver.getPageSource());
@@ -159,7 +160,7 @@ def getDocument(driver){
for(def ele : proviceEles){
def provice = ele.text()
def value = ele.attr("data-proviceid")
if(value && provice!="北京市" && provice!="天津市"){
if(value && provice!="北京市" && provice!="天津市" && provice!="吉林省" && provice!="辽宁省" && provice!="内蒙古自治区" && provice!="山西省" && provice!="河北省" && provice!="江苏省" && provice!="黑龙江省"){
provinceMap.put(provice, value)
}
}
@@ -167,8 +168,8 @@ def getDocument(driver){
return provinceMap;
}
def batchParse(def urls){
WebDriver driver = new ChromeDriver()
def batchParse(def urls, driver){
def ret = []
for(def map :urls){
def doc = getDocument(map.url, driver)
@@ -280,7 +281,7 @@ def parse(province, city, url,doc){
result.lookCount = JQueryUtils.selectText(".look_count span", doc)
def startDateStr = JQueryUtils.selectText("span:contains(ÅÄÂô»á¶¨ÓÚ)", doc)
if(startDateStr){
if(startDateStr && startDateStr.endsWith("定于") && startDateStr.endsWith("日")){
startDateStr = startDateStr.toString().substring(startDateStr.indexOf("¶¨ÓÚ")+2, startDateStr.indexOf("ÈÕ")+1).trim()
}
@@ -329,6 +330,13 @@ def parse(province, city, url,doc){
result.tradeAvgPrice = (int) (result.tradePrice / result.area)
}
if(result.finishDate){
result.finishDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.finishDate, "2000-01-01")
}
if(result.startDate){
result.startDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.startDate, "2050-01-01")
}
result.content = DataUtil.zip(doc.toString())
result.introduce = JQueryUtils.select([".d-block table:eq(1)"], doc).toString()
result.url = url
@@ -16,6 +16,27 @@ import java.util.regex.Matcher
import java.util.regex.Pattern
class DataUtil {
static def checkMinDate(curDate, checkDate){
if(curDate){
def minDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(checkDate,"yyy-MM-dd")
if(curDate.getTime()<minDate.getTime()){
return null
}
}
return curDate
}
static def checkMaxDate(curDate, checkDate){
if(curDate){
def maxDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(checkDate,"yyy-MM-dd")
if(curDate.getTime()>maxDate.getTime()){
return null
}
}
return curDate
}
static def getAreaNumber(def input, def maxArea) {
Pattern pattern = Pattern.compile('\\d+(.\\d+)?(ƽ·½Ã×|©O|m2|·½Ã×|ƽ)')
Matcher matcher = pattern.matcher(input)
@@ -112,10 +133,10 @@ class DataUtil {
if (list.size() >= 1) {
String s1 = list.get(0)
if (s1.contains("ÍòÔª")) {
s1 = s1.toString().replace("起拍价:", "").replace("万元", "").replace(",", "").replace(",", "")
s1 = s1.toString().replace("起拍价:", "").replace("万元", "").replace(",", "").replace(",", "").replace(" ","")
return (int) (s1.toDouble() * 10000)
} else {
s1 = s1.toString().replace("起拍价:", "").replace("元", "").replace(",", "").replace(",", "")
s1 = s1.toString().replace("起拍价:", "").replace("元", "").replace(",", "").replace(",", "").replace(" ","")
if (s1.isInteger()) {
return s1.toInteger()
} else {
@@ -0,0 +1,348 @@
package com.fangjia.script.gj.paimai
import com.fangjia.core.RegionData
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import javafx.scene.input.DataFormat
import net.sf.json.JSONObject
import org.jsoup.Jsoup
import org.openqa.selenium.By
import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver
class IcbcCrawl {
DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai")
static main(args) {
def task = new IcbcCrawl();
task.run();
}
def run(){
def url = "https://gf.trade.icbc.com.cn/searchproducts/pv.jhtml?resultLevel=1&FilterObjectName=%E8%B5%84%E4%BA%A7%E5%A4%84%E7%BD%AE%E5%B9%B3%E5%8F%B0&filterObjectId=8&category_Type=3&categoryId_level=&subjectChildType=1#this"
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
driver.get(url)
driver.findElementById("category_SM310000000000000006").click()
Thread.sleep(3000)
def urls = getAllDetailLinks(driver)
println "共:"+urls.size()
detailCrawling(urls, driver)
}
def getAllDetailLinks(driver){
def stop = false
def urls = []
while (!stop){
def doc = Jsoup.parse(driver.getPageSource());
def eles = JQueryUtils.select(".tabContent5 .ymzs_sp", doc)
if(eles){
for(def e : eles){
def href = e.select(".li_2_name a").attr("href")
urls << "https://gf.trade.icbc.com.cn"+href
}
def nextPage = JQueryUtils.select("a:contains(下一页)", doc)
if(nextPage){
driver.findElement(By.linkText("下一页")).click();
}else{
stop = true
}
}else{
stop = true
}
// if(urls.size()>=10){
// stop = true
// }
}
return urls;
}
def detailCrawling(def urls, driver){
def rets = []
for(def url : urls){
driver.get(url)
Thread.sleep(5000)
def doc = Jsoup.parse(driver.getPageSource());
def result = parser(doc, url)
if (result) {
rets << result
println rets.size()
}
if (rets.size() % 100 == 0) {
def add= saveData(rets)
println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")+" add:"+add
rets = []
}
}
if(rets.size()>0){
saveData(rets)
print "完"
}
}
def parser(doc, url){
def result = [:]
if(doc){
def title = JQueryUtils.selectText("#prodnameb", doc)
def id = url.substring(url.indexOf("="), url.length())
if(title){
def cityStr = JQueryUtils.selectText("dl:contains(所在城市:) li", doc)
if(cityStr && cityStr.contains("...")){
def cityEles = JQueryUtils.select("dl:contains(所在城市:) li", doc)
if(cityEles){
cityStr = cityEles[0].attr("title")
}
}
if(cityStr && cityStr.contains("省")){
result.province = cityStr.substring(0, cityStr.indexOf("省"))
result.city = cityStr.substring(cityStr.indexOf("省")+1, cityStr.length())
}else{
if(cityStr.contains("自治区")){
result.province = cityStr.substring(0, cityStr.indexOf("自治区")+3)
result.city = cityStr.substring(cityStr.indexOf("自治区")+3, cityStr.length())
}else{
result.city = cityStr.substring(0, cityStr.indexOf("市"))
}
}
if(!result.city){
return null
}
result.title = title
def rMap = OpenApiUtils.matchDistrict(result.city, result.title, null, true)
if (rMap) {
result.matchCity = rMap.city
result.matchRegion = rMap.region
result.matchName = rMap.name
result.matchAddress = rMap.address
result.houseNum = rMap.house_num
result.roomNum = rMap.room_num
}
result.status = JQueryUtils.selectText("#message", doc).replace("本场竞买","")
def t1 = JQueryUtils.selectText("#pricep", doc)
if(t1 && t1.contains("原因")){
result.statusDesc = t1.substring(t1.indexOf("原因")+3, t1.toString().length())
}
result.type = JQueryUtils.selectText("dl:contains(类  型:) dd", doc)
def enrollCount = JQueryUtils.selectText(["#signUpNum"], doc)
if(enrollCount) {
result.enrollCount = enrollCount.toInteger()
}
def lookCount = JQueryUtils.selectText(["#visitorCount"], doc)
if(lookCount) {
result.lookCount = lookCount.toInteger()
}
def callCount = JQueryUtils.selectText(["#remindPPNum"], doc)
if(callCount) {
result.callCount = callCount.toInteger()
}
def curPriceStr = JQueryUtils.selectText(["#new_price_key"], doc)
if(curPriceStr) {
curPriceStr = curPriceStr.replace(",", "").replace("元", "").trim()
if(curPriceStr.isInteger()) {
result.curPrice = curPriceStr.toInteger()
}else {
result.curPrice = (int)(curPriceStr.toDouble())
}
}
def startPriceEles = JQueryUtils.select(["dl:contains(起始价格:) dd"], doc)
if(startPriceEles) {
def startPriceStr = startPriceEles[0].attr("title")
if(startPriceStr){
startPriceStr = startPriceStr.replace(",", "").replace("¥","")
if(startPriceStr.isInteger()) {
result.startPrice = startPriceStr.toInteger()
}else {
result.startPrice = (int)(startPriceStr.toDouble())
}
}
}
def addPriceEles = JQueryUtils.select(["dl:contains(加价幅度:) dd"], doc)
if(addPriceEles){
def addPriceStr = addPriceEles[0].attr("title")
if(addPriceStr){
addPriceStr = addPriceStr.replace(",", "").replace("¥","")
if(addPriceStr) {
result.addPrice = addPriceStr.toDouble().toInteger()
}
}
}
def evaluationPriceEles = JQueryUtils.select(["dl:contains(评估价格:) dd"], doc)
if(evaluationPriceEles){
def evaluationPriceStr = evaluationPriceEles[0].attr("title")
if(evaluationPriceStr){
evaluationPriceStr = evaluationPriceStr.replace(",", "").replace("¥","")
if(evaluationPriceStr.isInteger()) {
result.evaluationPrice = evaluationPriceStr.toDouble().toInteger()
}else {
result.evaluationPrice = (int)(evaluationPriceStr.toDouble())
}
}
}
def depositPriceEles = JQueryUtils.select(["dl:contains(加价幅度:) dd"], doc)
if(depositPriceEles){
def v = depositPriceEles[0].attr("title")
if(v){
v = v.replace(",", "").replace("¥","")
if(v.isInteger()) {
result.depositPrice = v.toDouble().toInteger()
}else {
result.depositPrice = (int)(v.toDouble())
}
}
}
result.delayWeekly = JQueryUtils.selectText(["dl:contains(延时周期:) dd div:eq(0)"], doc)
result.person = JQueryUtils.selectText(["dl:contains(联系人:) dd"], doc)
result.court = JQueryUtils.selectText("#goToCourt", doc)
def mobileEles = JQueryUtils.select("dl:contains(联系电话:) dd", doc)
if(mobileEles){
result.mobile = mobileEles[0].attr("title")
}
def s1 = JQueryUtils.selectText("#div_announcement_gfword", doc)
if(s1) {
def start = s1.indexOf("法院将于")
def end = s1.indexOf("止")
if(start>-1 && end>-1 && end>start){
s1 = s1.substring(s1.indexOf("法院将于"), s1.indexOf("止"))
}
def dateRangs = PaimaiDataUtil.getDateRange(s1)
if(dateRangs && dateRangs.size()==2) {
result.startDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[0], "yyyy年MM月dd日HH时")
result.finishDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[1], "yyyy年MM月dd日HH时")
}
}
def trs = JQueryUtils.select([".rygtable table tr"], doc)
if(trs) {
def detailMap = [:]
for(def tr : trs) {
def d_tds = tr.select("td")
if(d_tds && d_tds.size()>=2) {
if(d_tds.size()==3) {
if(d_tds[2].text() && d_tds[2].text()!="/"){
detailMap.put(d_tds[1].text(), d_tds[2].text())
}
}else {
if(d_tds[1].text() && d_tds[1].text()!="/") {
detailMap.put(d_tds[0].text(), d_tds[1].text())
}
}
}
}
if(detailMap) {
result.detailMap = DataUtil.filterChar(detailMap)
}
if(!result.address) {
result.address = detailMap.get("拍品名称")
if(!result.address) {
result.address = detailMap.get("标的名称")
}
}
}
def imgs =JQueryUtils.select("#scroll li img", doc)
if(imgs) {
def imgLinks = []
for(def img : imgs) {
def href = img.attr("src")
if(href.contains("icbc.com")) {
imgLinks << "https:"+href
}
}
result.imgUrls = imgLinks
}
def detailStr = JQueryUtils.selectText(".detail_list", doc)
if(detailStr){
def area = DataUtil.getAreaNumber(detailStr, 5000)
if(area) {
result.area = area.toDouble()
if(result.tradePrice) {
result.tradeAvgPrice = (int)(result.tradePrice/area.toDouble())
}
}
}
result.content = com.fangjia.core.util.DataUtil.zip(doc.toString())
result.introduce = detailStr.toString()
result.url = url
result.mDate = new Date()
result.source = "融e够"
return result;
}
}
return null
}
def getCallCount(id){
def url= "https://gf.trade.icbc.com.cn/wholeSaleProd/visitorCount.jhtml?prodId="+id
def s = ABuYunUtil.doPost(url, null, null, "utf-8", true)
if(s){
JSONObject jsonObject = JSONObject.fromObject(s);
println jsonObject
}
}
def saveData(def results){
def add = 0
for(def ret : results){
def q = new BasicDBObject()
q.put("url", ret.url)
def dbo = paimaiResultColl.findOne(q);
DBObject obj = new BasicDBObject();
ret.each{k,v->
obj.put(k,v)
}
if(dbo){
obj.m_date = new Date()
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
}else{
add++
obj.c_date = new Date()
obj.m_date = new Date()
try{
paimaiResultColl.save(obj)
}catch(Exception e){
println obj.url
}
}
}
return add
}
}
@@ -39,8 +39,6 @@ class JdPaimaiCrawl {
println "¹²£º"+ids.size()
parserList( existsIds,ids, driver)
}
def getList(driver){
@@ -478,6 +476,13 @@ class JdPaimaiCrawl {
}
}
if(result.finishDate){
result.finishDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.finishDate, "2000-01-01")
}
if(result.startDate){
result.startDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.startDate, "2050-01-01")
}
result.content = com.fangjia.core.util.DataUtil.zip(doc.toString())
result.introduce = JQueryUtils.select([".floor-content div:eq(2) table"], doc).toString()
result.url = url
@@ -37,7 +37,7 @@ public class LianjiaCrawlByChrome {
task.run()
// task.run1()
task.addDealRegion();
// task.standardRegion()
task.standardRegion()
}
def run1(){
@@ -227,6 +227,10 @@ public class LianjiaCrawlByChrome {
break;
}
}
if(page>=100){
stop = true;
break;
}
page++;
}else{
stop = true;
@@ -416,10 +420,11 @@ public class LianjiaCrawlByChrome {
def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
def cityMap = [:]
def citys= ["낀뉴","섈주"]
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
if (!href.contains("fang.lianjia") && e.text().contains("¸£ÖÝ")) {
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
cityMap.put(e.text(), href)
}
}
@@ -4,9 +4,12 @@ import com.alibaba.fastjson.JSON
import com.fangjia.core.util.DataUtil
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.rabbitmq.client.impl.AMQImpl
import org.jsoup.Jsoup
import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver
@@ -19,9 +22,91 @@ class PaimaiTest {
DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai")
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
static main(args) {
def task = new PaimaiTest();
task.run();
// task.run();
// task.stat()
// task.updatePaimai()
task.updateFieldPaimai()
}
def updateFieldPaimai(){
def q = new BasicDBObject()
q.put("city", new BasicDBObject('$in',["重庆","重庆市"]))
def index = 0
def dbcursor = paimai43_COLL.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
for(def obj : dbcursor) {
index++
println index
if(obj.depositPrice){
obj.put("depositPrice", obj.depositPrice.toInteger())
paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj))
}
}
}
def updatePaimai(){
def q = new BasicDBObject()
def dbcursor = paimaiResultColl.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def index =0
for(def d : dbcursor){
index++
println index
def query = new BasicDBObject()
query.put("url", d.url)
def dbo = paimai43_COLL.findOne(query);
DBObject obj = new BasicDBObject();
d.each{k,v->
obj.put(k,v)
}
if(dbo){
obj.m_date = new Date()
paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj))
}else{
obj.c_date = new Date()
obj.m_date = new Date()
try{
paimai43_COLL.save(obj)
}catch(Exception e){
println obj.url
}
}
}
}
def stat(){
def q = new BasicDBObject()
q.put("city", new BasicDBObject('$in', ["重庆","重庆市"]))
def dbcursor = paimaiResultColl.find(q)
def keys = []
for(def db : dbcursor){
if(db.matchCity){
def key = db.matchCity +"##"+db.matchRegion+"##"+db.matchName;
if(!keys.contains(key)){
keys << key
}
// try{
// def jsonObject = OpenApiClient.addressMatch("重庆", db.address, "property", false, null, null)
// if(jsonObject != null && jsonObject.precise!=-1) {
// db.matchCity = jsonObject.get("city")
// db.matchRegion = jsonObject.get("district")
// db.matchName = jsonObject.get("name")
// paimaiResultColl.update(new BasicDBObject("_id", db._id),new BasicDBObject('$set',db))
// println db.address+" "+db.matchName
// }
// }catch(Exception e){
// println "error "+db.address
// }
}
}
println keys.size()
}
def run(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
@@ -1,7 +1,12 @@
package com.fangjia.script.gj.paimai
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.thirdPriceCrawl.CrawalUtil
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.util.DateUtils
import org.jsoup.Jsoup
import org.openqa.selenium.By
import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver
class TaobaoPaimaiCrawl {
static main(args) {
@@ -9,55 +14,432 @@ class TaobaoPaimaiCrawl {
task.run();
}
def run(){
def seedUrl = "https://sf.taobao.com/item_list.htm?spm=a213w.7398504.filter.86.675028eabFDx9r&category=50025969&auction_source=0&city=%C8%FD%D1%C7&sorder=2&st_param=-1&auction_start_seg=-1"
def regionUrls = regionCrawl(seedUrl);
def pageUrls = pageUrlCrawl(regionUrls)
detailCrawl(pageUrls)
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
// def url = "https://sf.taobao.com/list/50025969__-1.htm?spm=a213w.7398504.filter.62.73e954b3fMWNLd&auction_source=0&st_param=-1&auction_start_seg=-1"
def url = "https://sf.taobao.com/list/50025969__2.htm?spm=a213w.7398504.filter.65.101554b3Nynwwp&auction_source=0&st_param=-1&auction_start_seg=-1"
WebDriver driver = new ChromeDriver()
getAllCityRegions(driver, url);
}
def regionCrawl(def seedUrl){
def document = CrawalUtil.getDocumentByABuYun(seedUrl, "", "gbk");
def regionUrls = []
if(document){
def eles = JQueryUtils.select(".small-subcondion a", document)
if(eles){
for(def el : eles){
def reiongUrl = el.attr("href").trim()
if(reiongUrl.startsWith("//")){
regionUrls << "http:"+reiongUrl
def getAllCityRegions(driver, url){
def provinces = getAllProvince(driver, url)
def citys = ["天津","上海","北京","重庆"]
for(def province : provinces){
driver.findElement(By.xpath("//a[contains(text(), '${province}')]")).click()
def doc = Jsoup.parse(driver.getPageSource());
def cityEles= JQueryUtils.select(".unfold .J_FilterCity a")
for(def cityEle : cityEles){
def city = cityEle.text()
if(citys.contains(city)){
driver.findElement(By.xpath("//a[contains(text(), '${province}')]")).click()
doc = Jsoup.parse(driver.getPageSource());
driver.findElement(By.cssSelector(".J_FilterCity a")).click()
doc = Jsoup.parse(driver.getPageSource());
def regionEles= JQueryUtils.select(".J_SubCondition .small-subcondio ul li a")
for(def r : regionEles){
def m = [:]
m.province = province
m.city = city
m.region = r.text()
def urls = crawlLinks(m, driver)
parse(m, driver, urls)
}
}else{
def m = [:]
m.province = province
m.city = city
def urls = crawlLinks(m, driver)
parse(m, driver, urls)
}
}
}
return regionUrls;
}
def pageUrlCrawl(regionUrls){
def crawlLinks(def m, def driver){
def stop = false
def urls = []
for(def url : regionUrls){
def document = CrawalUtil.getDocumentByABuYun(url, "", "gbk");
if(document){
def pages = JQueryUtils.selectText(".page-total",document)
if(pages){
pages = pages.toInteger()
for(def page=1;page<pages;page++){
urls << url+"&page="+page
while(!stop){
def doc = Jsoup.parse(driver.getPageSource());
def details = JQueryUtils.select(".sf-pai-item-list li a", doc)
for(def d : details){
urls << "https:"+d.attr("href")
}
def nextEle = JQueryUtils.select(".next", doc)
if(nextEle){
driver.findElement(By.cssSelector(".next")).click()
}
}
return urls
}
def getAllProvince(driver, url){
driver.get(url);
Thread.sleep(100000)
def doc = Jsoup.parse(driver.getPageSource());
def eles= JQueryUtils.select("li:contains(标的物类型) .condition li a", doc)
def provinces = []
for(def e : eles){
def province = e.text()
provinces << province
}
return provinces
}
def parse(m, driver, urls){
def results = []
for(def url : urls) {
driver.get(url);
def doc = Jsoup.parse(driver.getPageSource());
def result = [:]
if (doc) {
def title = JQueryUtils.selectText([".pm-main h1"], doc)
def statusDesc = JQueryUtils.selectText([".bid-fail"], doc)
result.province = m.province
result.city = m.city
if (m.region) {
result.region = m.region
}
if (title) {
def addr = title.toString().replace("本场拍卖已", "").replace(" !", "")
if (addr && addr.indexOf("室") > -1) {
result.address = addr.substring(0, addr.indexOf("室") + 1)
} else {
result.address = addr
}
if (result.city) {
if (result.address.contains(city + "市")) {
result.address = result.address.substring(result.address.indexOf(city) + city.length() + 1)
} else {
result.address = result.address.substring(result.address.indexOf(city) + city.length())
}
}
if (result.address.indexOf("】") > -1) {
result.address = result.address.substring(result.address.indexOf("】") + 1)
}
if (result.city) {
def rMap = OpenApiUtils.matchDistrict(result.city, result.title, null, true)
if (rMap) {
result.matchCity = rMap.city
result.matchRegion = rMap.region
result.matchName = rMap.name
result.matchAddress = rMap.address
result.houseNum = rMap.house_num
result.roomNum = rMap.room_num
}
}
}
if (statusDesc) {
def status = statusDesc.substring(statusDesc.indexOf("本场拍卖") + 5, statusDesc.length() - 1)
result.status = DataUtil.getPaiMaiType(status)
result.statusDesc = statusDesc
}
def statusEle = JQueryUtils.select(".status-tip", doc)
if (statusEle) {
def sEle = JQueryUtils.select(".status-ing", doc)
if (sEle) {
result.status = "进行中"
} else {
result.status = "即将开始"
}
}
def curPrice = JQueryUtils.selectText(["#sf-price"], doc)
if (curPrice) {
if (curPrice.contains("当前价")) {
curPrice = JQueryUtils.selectText(["#sf-price .J_Price em:eq(0)"], doc)
if (curPrice) {
curPrice = curPrice.replace(",", "")
if (curPrice.isInteger()) {
result.curPrice = curPrice.toInteger()
} else {
result.curPrice = (int) (curPrice.toDouble())
}
}
} else if (curPrice.contains("成交价")) {
def tradePrice = JQueryUtils.selectText(["#sf-price .J_Price em:eq(0)"], doc)
if (tradePrice) {
tradePrice = tradePrice.replace(",", "")
if (tradePrice.isInteger()) {
result.tradePrice = tradePrice.toInteger()
} else {
result.tradePrice = (int) (tradePrice.toDouble())
}
}
} else if (curPrice.contains("起拍价")) {
def startPrice = JQueryUtils.selectText(["#sf-price .J_Price em:eq(0)"], doc)
if (startPrice) {
startPrice = startPrice.replace(",", "")
if (startPrice.isInteger()) {
result.startPrice = startPrice.toInteger()
} else {
result.startPrice = (int) (startPrice.toDouble())
}
}
}
}
def finishStr = JQueryUtils.selectText([".pm-bid-eyebrow .over-title"], doc)
if (finishStr) {
if (finishStr.contains("结束时间")) {
finishStr = JQueryUtils.select([".pm-main .J_TimeLeft"], doc).text()
if (finishStr) {
result.finishDate = DateUtils.parseFromTightByPattern(finishStr, "yyyy/MM/dd HH:mm:ss")
}
} else if (finishStr.contains("成交时间")) {
finishStr = JQueryUtils.select([".pm-main .J_TimeLeft"], doc).text()
if (finishStr) {
result.finishDate = DateUtils.parseFromTightByPattern(finishStr, "yyyy/MM/dd HH:mm:ss")
}
}
}
def enrollCount = JQueryUtils.selectText([".pm-remind .J_Applyer"], doc)
if (enrollCount) {
result.enrollCount = enrollCount.toInteger()
}
def callCount = JQueryUtils.selectText(["#J_NotifyNum"], doc)
if (callCount) {
result.callCount = callCount.toInteger()
}
def lookCount = JQueryUtils.selectText(["#J_Looker"], doc)
if (lookCount) {
result.lookCount = lookCount.toInteger()
}
def tds = JQueryUtils.select(["#J_HoverShow td"], doc)
if (tds && tds.size() > 0) {
for (def td : tds) {
def str = JQueryUtils.selectText("span:eq(0)", td).replace(" ", "")
if (str) {
if (str.contains("变卖价")) {
def str1 = td.select(".J_Price").text()
def str2 = td.select(".J_Price .J_Price-tip").text()
def p = str1.replace(str2, "").replace(",", "")
if (p) {
if (p.isInteger()) {
result.sellOffPrice = p.toInteger()
} else {
result.sellOffPrice = (int) (p.toDouble())
}
}
} else if (str.contains("加价幅度")) {
def str1 = td.select(".J_Price").text()
def str2 = td.select(".J_Price .J_Price-tip").text()
def p = str1.replace(str2, "").replace(",", "")
if (p) {
if (p.isInteger()) {
result.addPrice = p.toInteger()
} else {
result.addPrice = (int) (p.toDouble())
}
}
} else if (str.contains("类型")) {
result.type = td.select(".pay-type").text()
} else if (str.contains("保证金")) {
def str1 = td.select(".J_Price").text()
def str2 = td.select(".J_Price .J_Price-tip").text()
def p = str1.replace(str2, "").replace(",", "")
if (p) {
if (p.isInteger()) {
result.depositPrice = p.toInteger()
} else {
result.depositPrice = (int) (p.toDouble())
}
}
} else if (str.contains("变卖周期")) {
result.sellOffWeekly = td.select("span:eq(1)").text().replace(": ", "")
} else if (str.contains("竞价周期")) {
result.pricePkWeekly = td.select("span:eq(1)").text().replace(": ", "")
} else if (str.contains("优先购买权")) {
result.preemptivePerson = td.select("span:eq(1)").text().replace(": ", "")
} else if (str.contains("变卖预缴款")) {
def str1 = td.select("span:eq(1) .J_Price").text()
def str2 = td.select("span:eq(1) .J_Price .J_Price-tip").text()
def p = str1.replace(str2, "").replace(",", "")
if (p) {
if (p.isInteger()) {
result.sellOffAdvancePay = p.toInteger()
} else {
result.sellOffAdvancePay = (int) (p.toDouble())
}
}
} else if (str.contains("评估价")) {
def str1 = td.select(".J_Price").text()
def str2 = td.select(".J_Price .J_Price-tip").text()
def p = str1.replace(str2, "").replace(",", "")
if (p) {
if (p.isInteger()) {
result.evaluationPrice = p.toInteger()
} else {
result.evaluationPrice = (int) (p.toDouble())
}
}
} else if (str.contains("延时周期")) {
result.delayWeekly = td.select("span:eq(1)").text().replace(": ", "")
} else if (str.contains("起拍价")) {
def str1 = td.select(".J_Price").text()
def str2 = td.select(".J_Price .J_Price-tip").text()
def p = str1.replace(str2, "").replace(",", "")
if (p) {
if (p.isInteger()) {
result.startPrice = p.toInteger()
} else {
result.startPrice = (int) (p.toDouble())
}
}
}
}
}
}
result.rule = JQueryUtils.selectText(".pai-info .pai-content", doc)
def trs = JQueryUtils.select(["#J_desc table tr"], doc)
if (trs) {
def detailMap = [:]
for (def tr : trs) {
def d_tds = tr.select("td")
if (d_tds && d_tds.size() >= 2) {
if (d_tds.size() == 3) {
detailMap.put(d_tds[1].text(), d_tds[2].text())
} else {
detailMap.put(d_tds[0].text(), d_tds[1].text())
}
} else {
if (d_tds[0] && d_tds[0].text().contains("拍品介绍")) {
detailMap.put("拍品介绍", d_tds[0].text().replace("拍品介绍 ", ""))
}
}
}
if (detailMap) {
result.detailMap = DataUtil.filterChar(detailMap)
}
if (!result.address) {
result.address = detailMap.get("拍品名称")
if (!result.address) {
result.address = detailMap.get("标的名称")
}
}
}
def imgs = JQueryUtils.select([".slide-bigpic img"], doc)
if (imgs) {
def imgLinks = []
for (def img : imgs) {
def href = img.attr("src")
if (href.contains("http")) {
imgLinks << href
} else {
imgLinks << url.substring(0, url.indexOf(":") + 1) + href
}
}
result.imgUrls = imgLinks
}
}
//面积
def article = JQueryUtils.selectText(["#J_ItemNotice"], doc)
if (article) {
def area = DataUtil.getAreaNumber(article, 5000)
if (!area) {
article = JQueryUtils.selectText(["#J_NoticeDetail"], doc)
if (article) {
area = DataUtil.getAreaNumber(article, 5000)
}
}
if (!area) {
article = JQueryUtils.selectText(["body"], doc)
if (article) {
area = DataUtil.getAreaNumber(article, 5000)
}
}
if (area) {
result.area = area.toDouble()
if (result.tradePrice) {
result.tradeAvgPrice = (int) (result.tradePrice / area.toDouble())
}
}
}
}
return urls;
}
def detailCrawl(pageUrls){
if(pageUrls){
for(def pageUrl : pageUrls){
def document = CrawalUtil.getDocumentByABuYun(pageUrl, "", "gbk");
if(document){
def pages = JQueryUtils.select(".page-total", document)
def s1 = JQueryUtils.selectText(["#J_NoticeDetail"], doc)
if (s1) {
def dateRangs = DataUtil.getDateRange(s1)
if (dateRangs) {
for (def j = 0; j < dateRangs.size(); j++) {
def date = dateRangs[j]
def pattern = null
if (date.endsWith("日时") || date.endsWith("日")) {
pattern = "yyyy年MM月dd日"
} else {
def ind = date.indexOf("日")
if (date.indexOf("时") == -1 && date.toString().length() > ind + 1) {
date = date + "时"
}
pattern = "yyyy年MM月dd日HH时"
}
if (j == 0) {
result.startDate = DateUtils.parseFromTightByPattern(date, pattern)
} else {
if (!result.finishDate) {
result.finishDate = DateUtils.parseFromTightByPattern(date, pattern)
}
}
}
}
if (!result.startPrice) {
result.startPrice = DataUtil.getStartPrice(s1)
}
}
result.content = com.fangjia.core.util.DataUtil.zip(doc.toString())
def estateType = DataUtil.findEstateType(s)
if (estateType) {
result.matchEstateType = estateType
}
result.introduce = JQueryUtils.select(["#J_desc table"], doc).toString()
result.url = url
result.mDate = new Date()
results << result
}
return results;
}
@@ -1,5 +1,6 @@
package com.fangjia.script.loudong
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
@@ -16,8 +17,15 @@ class CqfdcChromeCrawl {
static main(args) {
def task = new CqfdcChromeCrawl();
task.run()
// def task = new CqfdcChromeCrawl();
// task.run()
def name = "E3뵀짜" //69눙(방뺍C-7뵀짜) 방뺍5-5뵀짜
def s = DataHandleUtils.findMatchPattern(name, "[\\w+]?-?\\w+[#|눙|떠|뵀]짜?.*?")
println s
def unitNumStr = DataHandleUtils.findMatchPattern(name, "\\d+데禱")
println unitNumStr
}
def run(){
@@ -1,20 +1,49 @@
package com.fangjia.script.loudong
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBObject
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher
import java.util.regex.Pattern
class CqfdcMatch {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", lineUrl)
static DBCollection loudongFdcConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong_cqfdc", "192.168.0.235", 27018)
static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong", "192.168.0.235", 27018)
public static final DBCollection LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia_base", "loudong", lineUrl)
public static final DBCollection HOUSE_LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", lineUrl)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
static main(args) {
def task = new CqfdcMatch();
// task.run()
// task.stat()
task.export2()
// task.export2()
// task.matchLoudong()
task.addLoudong()
// task.addFdcLoudong()
}
def export(){
def q = new BasicDBObject()
@@ -139,6 +168,378 @@ class CqfdcMatch {
return rets;
}
def addHouseLoudong(){
def q = new BasicDBObject()
def dbcursor = loudongConn.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def keys = []
def index = 0
for(def d : dbcursor) {
index++
println index
if(d.room_num && (d.room_num.contains("商") || d.room_num.contains("社区用房"))){
continue
}
if(d.area.toDouble()<=25){
continue
}
def newRegion = RegionData.formatRegion(d.city, d.region)
if (newRegion) {
def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?")
def house_num = null
if (houseNumStr) {
house_num = houseNumStr.replace("号楼", "").replace("幢", "").replace("栋", "").replace("号", "")
}
if (house_num) {
def unitNum = DataHandleUtils.findMatchPattern(d.house_num, "\\d+单元")
if (unitNum) {
unitNum = unitNum.replace("单元", "")
}
def key = d.city + "##" + newRegion + "##" + d.district_name + "##" + house_num+ "##" + unitNum +"##"+d.room_num
if(!keys.contains(key) && d.room_num) {
keys << key
def count = findHouseLoudongDatas(d.city, newRegion, d.district_name, house_num, unitNum, d.room_num)
if(!count){
def dist = null
if(d.match_city){
dist = CommonDatautils.findDistrict(d.match_city, d.match_region, d.match_name,"district")
}
def udata = new BasicDBObject()
udata.put("city", d.city)
udata.put("region", newRegion)
udata.put("name", d.district_name)
udata.put("house_num", house_num)
if(unitNum){
udata.put("house_num_unit", unitNum)
}
udata.put("room_num", d.room_num)
udata.put("house_area",d.area.toDouble())
udata.put("status", 0)
udata.put("from", "房谱网")
if(d.floor_heigth && d.floor_heigth.isNumber()){
udata.put("house_height", d.floor_heigth.toInteger())
}
if(d.floor && d.floor.isNumber()){
udata.put("house_floor", d.floor.toInteger())
}
if(dist){
udata.put("district_id", dist._id.toString())
udata.put("name_in_seaweed", dist.name)
}
if(d.address){
def address = AddressUtil.formatAddress(d.city, d.address)
if(address){
udata.put("address", address)
}
}
udata.put("c_date", new Date())
udata.put("m_date", new Date())
HOUSE_LOUDONG_LINE_COLL.save(udata)
}
}
}
}
}
}
def findHouseLoudongDatas(city, region, district_name, house_num, unitNum, room_num){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", district_name)
q.put("house_num", house_num)
if(unitNum){
q.put("unit_num", unitNum)
}
q.put("room_num", room_num)
def count = HOUSE_LOUDONG_LINE_COLL.count(q)
return count;
}
def findAllLoundongByHouseNum(){
def q = new BasicDBObject()
def dbcursor = loudongConn.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(5000)
def keys = []
for(def d : dbcursor){
def newRegion = RegionData.formatRegion(d.city, d.region)
if(newRegion){
def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?")
def house_num = null
if(houseNumStr){
house_num = houseNumStr.replace("号楼","").replace("幢","").replace("栋","").replace("号","")
}
if(house_num){
def unitNum = DataHandleUtils.findMatchPattern(d.house_num, "\\d+单元")
if(unitNum){
unitNum = unitNum.replace("单元","")
}
def key = d.city +"##"+newRegion+"##"+d.district_name+"##"+house_num
if(unitNum){
key = key +"##"+unitNum
}
if(!keys.contains(key)){
keys << key
def obj = findLoudongOnLine(d.city, newRegion, d.district_name, house_num, d.unit_num)
println d.district_name+" "+d.house_num+"==${house_num}==${unitNum}"
if(obj){
def udata = new BasicDBObject()
def areas = obj.areas
if(!areas.contains(d.area)){
areas << d.area
def forms = obj.from
if(!forms.contains("房谱网")){
forms << "房谱网"
}
obj.m_date = new Date()
def query = new BasicDBObject("_id", obj._id)
LOUDONG_LINE_COLL.update(query, new BasicDBObject('$set', obj))
}
}else{
def udata = new BasicDBObject()
udata.put("city", d.city)
udata.put("region", newRegion)
udata.put("name", d.district_name)
udata.put("house_num", house_num)
if(unitNum){
udata.put("unit_num", unitNum)
}
if(d.address){
def address = AddressUtil.formatAddress(d.city, d.address)
if(address){
udata.put("address", address)
}
}
udata.put("areas",[d.area])
udata.put("from",["房谱网"])
if(d.floor_heigth && d.floor_heigth.toString().toInteger()){
udata.put("floors", d.floor_heigth.toString().toInteger())
}
udata.put("c_date", new Date())
udata.put("status", 1)
LOUDONG_LINE_COLL.save(udata)
}
}
}
}
}
}
def addFdcHouseLoudong(){
def q = new BasicDBObject()
def dbcursor = loudongFdcConn.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(2000)
def keys = []
def index = 0
for(def d : dbcursor){
index++
println index
if(!d.use_type){
continue
}
if(!d.use_type.contains("住宅")){
continue
}
def newRegion = d.match_region
if(newRegion){
def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num_desc, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?")
def house_num = null
if(houseNumStr){
house_num = houseNumStr.replace("号楼","").replace("幢","").replace("栋","").replace("号","")
}
if(house_num){
def key = d.city +"##"+newRegion+"##"+d.name+"##"+house_num+"##"+d.unit_num+"##"+d.room_num
if(!keys.contains(key)) {
keys << key
def count = findHouseLoudongDatas(d.city, newRegion, d.name, house_num, d.unit_num, d.room_num)
if(!count){
def maxHeight = findMaxHeight(d.city, d.name, d.house_num_desc, d.unit_num)
def dist = null
if(d.match_city){
dist = CommonDatautils.findDistrict(d.match_city, d.match_region, d.match_name,"district")
}
def floor = null
if(d.room_num && d.room_num.contains("-")) {
def arrs = d.room_num.split("-")
floor = arrs[0]
}
Integer room = null
Integer hall = null
if(d.room_type && d.room_type.contains("室")){
def roomStr = d.room_type.substring(0,d.room_type.indexOf("室"))
if(roomStr){
roomStr = roomStr.replace("一","1").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("六","6").replace("七","7").replace("八","8").replace("九","9").replace("十","10")
room = roomStr.toInteger()
}
if(d.room_type.contains("厅")){
def hallStr = d.room_type.substring(d.room_type.indexOf("室")+1, d.room_type.indexOf("厅"))
if(hallStr){
hallStr = hallStr.replace("一","1").replace("两","2").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("六","6").replace("七","7").replace("八","8").replace("九","9").replace("十","10")
hall = hallStr.toInteger()
}
}
}
def udata = new BasicDBObject()
udata.put("city", d.city)
udata.put("region", newRegion)
udata.put("name", d.name)
udata.put("house_num", house_num)
if(d.unit_num){
udata.put("house_num_unit", d.unit_num)
}
udata.put("room_num", d.room_num)
udata.put("house_area",d.area.toDouble())
udata.put("status", 0)
udata.put("from", "重庆网上房地产")
udata.put("house_height", maxHeight)
if(room){
udata.put("room", room)
}
if(hall){
udata.put("hall", hall)
}
if(floor && floor.isNumber()){
udata.put("house_floor", floor.toInteger())
}
if(dist){
udata.put("district_id", dist._id.toString())
udata.put("name_in_seaweed", dist.name)
}
if(d.address){
def address = AddressUtil.formatAddress(d.city, d.address)
if(address){
udata.put("address", address)
}
}
udata.put("c_date", new Date())
udata.put("m_date", new Date())
HOUSE_LOUDONG_LINE_COLL.save(udata)
}
}
}
}
}
}
public String getMatchString(String text,String pattern){
Pattern p=Pattern.compile(pattern);
Matcher m=p.matcher(text);
if(m.find()){
return m.group(0)
}
return null;
}
def addFdcBaseLoudong(){
def q = new BasicDBObject()
def dbcursor = loudongFdcConn.find(q).batchSize(5000)
def keys = []
for(def d : dbcursor){
if(!d.match_region){
continue
}
def newRegion = d.match_region
def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num_desc, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?")
def house_num = null
if(houseNumStr){
house_num = houseNumStr.replace("号楼","").replace("幢","").replace("栋","").replace("号","")
}
if(house_num){
def key = d.city +"##"+newRegion+"##"+d.name+"##"+house_num
if(d.unit_num) {
key = key + "##" + d.unit_num
if (!keys.contains(key)) {
keys << key
def obj = findLoudongOnLine(d.city, newRegion, d.name, house_num, d.unit_num)
println d.name + " " + d.house_num_desc + "==${house_num}==${d.unit_num}"
def maxHeight = findMaxHeight(d.city, d.name, d.house_num_desc, d.unit_num)
if (obj) {
def areas = obj.areas
if (!areas.contains(d.area)) {
areas << d.area
def forms = obj.from
if (!forms.contains("重庆网上房地产")) {
forms << "重庆网上房地产"
}
obj.m_date = new Date()
def query = new BasicDBObject("_id", obj._id)
LOUDONG_LINE_COLL.update(query, new BasicDBObject('$set', obj))
}
} else {
def udata = new BasicDBObject()
udata.put("city", d.city)
udata.put("region", newRegion)
udata.put("name", d.name)
udata.put("house_num", house_num)
if (d.unit_num) {
udata.put("unit_num", d.unit_num)
}
if (d.address) {
def address = AddressUtil.formatAddress(d.city, d.address)
if (address) {
udata.put("address", address)
}
}
udata.put("areas", [d.area])
udata.put("from", ["重庆网上房地产"])
udata.put("floors", maxHeight)
udata.put("c_date", new Date())
udata.put("status", 1)
LOUDONG_LINE_COLL.save(udata)
}
}
}
}
}
}
def findMaxHeight(city, name,house_num, unit_num){
def q = new BasicDBObject()
q.put("city",city)
q.put("name",name)
q.put("house_num_desc",house_num)
q.put("unit_num",unit_num)
def dbcursor = loudongFdcConn.find(q)
def heights = []
for(def h : dbcursor){
if(h.nominal_floor && h.nominal_floor.isNumber()){
def height = h.nominal_floor.toInteger()
if(!heights.contains(height)){
heights << height
}
}
}
if(heights){
heights = heights.sort()
return heights[heights.size()-1]
}
return null
}
def findLoudongOnLine(city, region, name, house_num, unit_num){
def q = new BasicDBObject()
q.put("city",city)
q.put("region",region)
q.put("name",name)
q.put("house_num",house_num)
if(unit_num){
q.put("unit_num",unit_num)
}
def dbo = LOUDONG_LINE_COLL.findOne(q)
return dbo
}
def stat(){
def q = new BasicDBObject()
@@ -168,6 +569,135 @@ class CqfdcMatch {
}
def matchLoudong(){
def names = findAllLoudong("重庆")
print names.size()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(names.size())
def retMap = new ConcurrentHashMap()
for(def i=0;i<names.size();i++){
def key = names[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def arrs = key.split("#")
def jsonObject = OpenApiClient.addressMatch(arrs[0], arrs[2], "property", false, arrs[1], null)
if(jsonObject != null && jsonObject.precise!=-1) {
def m = [:]
m.id = jsonObject.get("id")
m.city = jsonObject.get("city")
m.region = jsonObject.get("district")
m.name = jsonObject.get("name")
def dist = CommonDatautils.findDistrictById(m.id)
if(dist){
m.sid = dist.sid
}
retMap.put(key, m)
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
println "匹配后:"+retMap.size()
updateLoudong(retMap)
println "修改完成!"
}
def updateLoudong(retMap){
def index = 0
retMap.each{key,m->
def arrs = key.split("#")
def q = new BasicDBObject("city", arrs[0])
q.put('region', arrs[1]);
q.put('name', arrs[2]);
def udata = new BasicDBObject()
udata.put("district_id", m.id.toString())
udata.put("name_in_seaweed", m.name)
udata.put("district_no", m.sid)
HOUSE_LOUDONG_COLL.update(q, new BasicDBObject('$set', udata), false, true);
index++
println index
}
}
def findAllLoudong(city){
def q = new BasicDBObject()
q.put("city", city)
q.put("district_id",new BasicDBObject('$exists', false))
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("region", 1)
keys.put("name", 1)
def dbcursor = HOUSE_LOUDONG_COLL.find(q).batchSize(5000)
def names = new CopyOnWriteArrayList()
for(def db : dbcursor){
def key = db.city +"#"+db.region+"#"+db.name
if(!names.contains(key)){
names << key
}
}
return names;
}
def addLoudong(){
// def stat = statBaseLoudong()
// println stat
// findAllLoundongByHouseNum();
def stat1 = statHouseLoudong()
println stat1
// addHouseLoudong()
}
def addFdcLoudong(){
// addFdcBaseLoudong()
addFdcHouseLoudong()
}
def statHouseLoudong(){
def q = new BasicDBObject()
q.put("city", "重庆")
def dbcursor = HOUSE_LOUDONG_LINE_COLL.find(q)
def total = dbcursor.size()
def ids = []
for(def d : dbcursor){
if(d.district_id && !ids.contains(d.district_id)){
ids << d.district_id
}
}
def map = [:]
map.rows_total = total
map.dist_count = ids.size()
return map
}
def statBaseLoudong(){
def q = new BasicDBObject()
q.put("city", "重庆")
def dbcursor = LOUDONG_LINE_COLL.find(q)
def total = dbcursor.size()
def ids = []
for(def d : dbcursor){
if(d.id_in_seaweed && !ids.contains(d.id_in_seaweed)){
ids << d.id_in_seaweed
}
}
def map = [:]
map.rows_total = total
map.dist_count = ids.size()
return map
}
}
@@ -12,7 +12,8 @@ import com.fangjia.script.util.DateUtils
class DingTalkUtils {
static main(String[] args) {
Date now = DateUtils.parseFromTightByPattern("2021-08-01", "yyyy-MM-dd") ;
// 要导出的月份第一天
Date now = DateUtils.parseFromTightByPattern("2022-03-01", "yyyy-MM-dd") ;
String startTime = DateUtils.addMonth(now, -1) + " 00:00:00"
String monthStartTime = DateUtils.formatDateByPattern(now, "yyyy-MM-dd HH:mm:ss")
@@ -0,0 +1,46 @@
package com.fangjia.script.temp.shenjian.geo
import com.fangjia.script.gj.location.Point
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.xh.InsidePolygon
import com.mongodb.BasicDBList
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "block", "192.168.0.85", 28017)
collection.find(new BasicDBObject()).forEach({ obj ->
if (obj.type == 'block') {
if (obj.region instanceof String) {
return;
}
BasicDBList reigon = (BasicDBList)obj.region
int count = 0
List<String> info = new ArrayList<>()
for (String s : reigon) {
info.add(s)
}
if (info.size() > 1) {
boolean find = false
for (String s : info) {
if (find) {
break
}
collection.find(new BasicDBObject("type", "region").append("name", s)).limit(1).forEach({obj2 ->
String border = obj2.border
String points = border.replaceAll(",", "-").replaceAll(";", "-")
List<com.fangjia.script.looplineinfo.Point> pointList = InsidePolygon.createPoint(points);
com.fangjia.script.looplineinfo.Point point = new com.fangjia.script.looplineinfo.Point(obj.lng, obj.lat)
if (InsidePolygon.aboutZone(point, pointList)) {
collection.update(new BasicDBObject("_id", obj._id),
new BasicDBObject('$set', new BasicDBObject("region", obj2.name)));
find = true;
}
})
}
} else {
collection.update(new BasicDBObject("_id", obj._id),
new BasicDBObject('$set', new BasicDBObject("region", info.get(0))));
}
}
})
@@ -0,0 +1,27 @@
package com.fangjia.script.temp.shenjian.poi
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.collections.CollectionUtils
import org.apache.commons.lang.StringUtils
def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
final DBCollection ZS_REGION_CONN = MongoUtils.getCollectionByName2("fangjia", "region_block", ZS_MONGODB_URL)
ZS_REGION_CONN.find(new BasicDBObject()).batchSize(1000).each {obj ->
def city = obj.city
def org_name = obj.org_name
List<AmapPoi> result = GaoDeUtils.searchPoi(city, org_name, org_name, null)
if (CollectionUtils.isNotEmpty(result)) {
def location = result.get(0).location
def lon = location.split(",")[0]
def lat = location.split(",")[1]
ZS_REGION_CONN.update(new BasicDBObject("_id", obj._id),
new BasicDBObject('$set',new BasicDBObject("location", new BasicDBObject("lon", lon).append("lat", lat))))
}
}
@@ -128,10 +128,10 @@ class SeaweedUtils {
*/
static formatHouseholdCount(def household_count) {
if (household_count && FieldValidatorUtils.checkValue(household_count) ) {
if (Long.valueOf(household_count) < 50) {
if (Integer.valueOf(household_count) < 50) {
return "" // »§Êý
}
return household_count
return household_count.toInteger()
}
}
@@ -40,13 +40,13 @@ class Esf58Crawl extends BaseBatch{
def static useDistKeys = []
static def totalCount = new AtomicLong()
def cityList = ["成都":"cd|102","杭州":"hz|79","重庆":"cq|37","苏州":"su|5","无锡":"wx|93", "深圳":"sz|4","郑州":"zz|342", "长沙":"cs|414","东莞":"dg|413",
def cityList = ["福州":"fz|304","成都":"cd|102","杭州":"hz|79","重庆":"cq|37","苏州":"su|5","无锡":"wx|93", "深圳":"sz|4","郑州":"zz|342", "长沙":"cs|414","东莞":"dg|413",
"괏땍":"bd|424","괏癸":"baoting|10367","켓퀼":"nn|845", "爛鹵":"yz|637","�庫":"st|783","揭鹵":"taizhou|693","뺨갛":"ha|968","纜쉭":"zhanjiang|791","쏩鹵":"jinzhou|2354","웹츠":"km|541","진鹵":"liuzhou|7133","凉헤":"zq|901","젯暾멍":"lyg|2049","휑亮":"rizhao|3177","끽돠":"changde|872",
"꾕鹵":"cangzhou|652", "홋鹵":"qz|291","헌陶":"qingyuan|7303", "鮫퀼":"xn|2052","潾屢":"zunyi|7620","瓏碌":"zhuzhou|1086","뱍捺":"gy|2015","纛捺":"yy|821","其鹵":"xz|471", "벗랑깝":"hrb|202","露끗":"yc|858", "읜렌":"lf|772","해뽈돎":"qhd|1078","쁨鹵":"huizhou|722","쭤捺":"luoyang|556",
"柯소鏤":"sjz|241","格覩":"ty|740","吉냘":"yancheng|613","斤捺":"xy|8694","켓념":"nanchong|2378","켓捺":"ny|592","衢쳔":"xm|606",
"축捺":"mianyang|1057","셌퀼":"jining|450","벳덱":"hd|572",
"뺨굇":"huaibei|9357","뇔鹵":"chuzhou|10266",
"福州":"fz|304","蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158",
"蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158",
"읊鹵":"lz|952","북례":"hf|837", "魁쨀컁펭":"xj|984","켓끗":"nc|669","�捺":"sy|188","鮫갛":"xa|483","藿렌":"wf|362",
"交빤":"wuhu|2045","땜賻":"dg|413","끽鹵":"cz|463","뤘�":"fs|222",
"켓繫":"nt|394","섈주":"jl|700","櫓�":"zs|771","막捺":"fy|2325",
@@ -192,37 +192,37 @@ class Esf58Crawl extends BaseBatch{
void process(Object args) {
//addContent("르鹵","https://fz.58.com/xiaoqu/", 70);
def citys = ["佛山","潍坊","长春","太原","南通","厦门","呼和浩特","常州","中山","德阳","镇江","泰安","银川","襄阳","江门","荆州","上海","衡阳","滨州","遂宁","鞍山","赣州","阜阳","枣庄","聊城","芜湖","吉林","德州","信阳","新乡","宿迁","沧州","桂林","包头","常德","乐山","遵义","眉山","株洲","驻马店","日照","许昌","成都","衡水","宜宾","泸州","营口","达州","宝鸡","广安","吉安","孝感","鄂尔多斯","十堰","清远","湘潭","上饶","漯河","邢台","平顶山","菏泽","宜春","承德","赤峰","延边","茂名","焦作","内江","韶关","梅州","郴州","黄冈","红河","绥化","娄底","丹东","邵阳","开封","阳江","黄石","周口","牡丹江","长治","咸宁","汉中","自贡","衢州","晋中","永州","北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏","重庆"]
for(def city : citys){
def str = cityList.get(city)
for(def i=0;i<1;i++){
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
if(districts){
def arrs = str.split("\\|")
def url = "https://"+arrs[0]+".58.com/xiaoqu/"
distMap.put(city, districts)
addCrawlKeys(city);
addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
}
}
}
// cityList.each{city, str->
// def citys = ["北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏"]
// for(def city : citys){
// def str = cityList.get(city)
// for(def i=0;i<1;i++){
// if(citys.contains(city)){
// def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
// if(districts){
// def arrs = str.split("\\|")
// def url = "https://"+arrs[0]+".58.com/xiaoqu/"
// distMap.put(city, districts)
//
// addCrawlKeys(city);
// addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
// }
// def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
// if(districts){
// def arrs = str.split("\\|")
// def url = "https://"+arrs[0]+".58.com/xiaoqu/"
// distMap.put(city, districts)
// addCrawlKeys(city);
// addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
// }
// }
// }
cityList.each{city, str->
for(def i=0;i<1;i++){
// if(citys.contains(city)){
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
if(districts){
def arrs = str.split("\\|")
def url = "https://"+arrs[0]+".58.com/xiaoqu/"
distMap.put(city, districts)
addCrawlKeys(city);
addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
}
// }
}
}
}
@@ -18,7 +18,7 @@ class SynchDistrictPrice {
// def sourceMap = ["fang.com":"搜房","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"]
//"fang.com":"搜房",
def sourceMap = ["ke.com":"贝壳找房","fang.com":"搜房","cityhouse.cn":"城市房产"]
def sourceMap = ["fang.com":"搜房","ke.com":"贝壳找房","5i5j.com":"我爱我家"]
static main(args) {
def task = new SynchDistrictPrice()
task.run()
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
for(def source :sources){
def count1 = findTotalCount(source, 202202);
def count2 = findTotalCount(source, 202203);
def count1 = findTotalCount(source, 202203);
def count2 = findTotalCount(source, 202204);
println source +" "+count1 +" "+count2
}
}
@@ -245,6 +245,48 @@ public class CommonUtils {
}
static void recalculationSqRent(String city, String region, String name, String estateType, Date calcDate = new Date(), boolean isTest = false, int calcType = 0, needRedrawTrend = true, def sync = false) {
calcDate = com.fangjia.core.util.DateUtil.truncateDay(calcDate)
if (estateType && estateType.contains("±ðÊû")) {
estateType = "±ðÊû"
} else if (estateType) {
estateType = "Ð¡Çø"
}
DBObject cmd = new BasicDBObject()
cmd.put("city", city)
cmd.put("region", region)
cmd.put("name", name)
cmd.put('estate_type', estateType)
cmd.put("calc_date", calcDate)
cmd.put("calc_type", 0)
println(cmd)
DBObject data = new BasicDBObject()
data.put("category", "square_rent")
data.put("priority", 3)
data.put("status", 0)
data.put("remark", 'week')
data.put("c_date", new Date())
if(isTest){
data.put("ip", "192.168.0.231")
COMMAND_COLL.update(cmd, new BasicDBObject('$set', data), true, false)
}else{
data.put("ip", ["fangjia-priceanalysis-0", "fangjia-priceanalysis-1"][new Random().nextInt(2)])
COMMAND_COLL.update(cmd, new BasicDBObject('$set', data), true, false)
}
if(sync){
while (COMMAND_COLL.count(cmd) != 0){
Thread.sleep(1000)
}
}
}
/**
* 重算小区均价
* @param city
@@ -305,6 +305,43 @@ public class DbUtils{
}
return districts
}
/**
* 查询附近一共里内小区的数量
*/
def static findAroundDistrictByRadius(BasicDBObject district, def radius){
def districts = []
if(hasGeo(district)){
def query = new BasicDBObject()
query.append("cat", "district")
query.append("city", district.city)
query.append("visible", 0)
DBObject sdq = new BasicDBObject();
def lat = district.lat / 10000000000.0
def lng = district.lng / 10000000000.0
sdq.put("sd.point", lat + "," + lng);
sdq.put("sd.sfield", "lat_lng");
sdq.put("sd.distance", String.valueOf((double) radius / 1000.0));
query.append('$sd', sdq)
def keys = new BasicDBObject()
keys.append("city", 1)
keys.append("region", 1)
keys.append("name", 1)
keys.append("lat", 1)
keys.append("lng", 1)
keys.append("lat2", 1)
keys.append("lng2", 1)
keys.append("estate_type2", 1)
DbUtils.SEAWEED_SLAVE.find(query, keys).each {BasicDBObject dbo->
if(!district._id || dbo._id != district._id){
districts << dbo
}
}
}
return districts
}
/**
* 查询小区均价
@@ -229,6 +229,93 @@ public class OpenApiClient {
return null
}
public static JSONObject searchLoudong(String id, String houseNum, String unitNum){
try{
Map<String, String> params = new HashMap<String, String>();
params.put("id", id);
params.put("houseNumber", houseNum);
params.put("unitNumber", unitNum);
params.put("token", TOKEN);
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/property/searchLoudong", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
while (result == null) {
result = HttpRequestUtil.getJsonContent("http://localhost:8080/fangjia-open/property/searchLoudong", params, "utf-8")
println("${id} ${houseNum} 程序调用异常")
Thread.sleep(6000)
}
if(result){
if (result.getInt("code") == 200) {
return result;
} else if (result.getInt("code") !=0) {
println("${id} ${houseNum}")
}
}
} catch (Exception e) {
println("${id} ${houseNum} ${e.getMessage()}")
e.printStackTrace();
}
return null
}
public static JSONObject searchHouseLoudong(String id, String houseNum, String unitNum, String roomNum){
try{
Map<String, String> params = new HashMap<String, String>();
params.put("id", id);
params.put("houseNumber", houseNum);
params.put("unitNumber", unitNum);
params.put("roomNumber", roomNum);
params.put("token", TOKEN);
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/property/searchHouseLoudong", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
while (result == null) {
result = HttpRequestUtil.getJsonContent("http://localhost:8080/fangjia-open/property/searchHouseLoudong", params, "utf-8")
println("${id} ${houseNum} 程序调用异常")
Thread.sleep(6000)
}
if(result){
if (result.getInt("code") == 200) {
return result;
} else if (result.getInt("code") !=0) {
println("${id} ${houseNum}")
}
}
} catch (Exception e) {
println("${id} ${houseNum} ${e.getMessage()}")
e.printStackTrace();
}
return null
}
public static JSONObject standard2Action(String city, String address) {
try {
Map<String, String> params = new HashMap<String, String>();
params.put("city", city);
params.put("address", address);
params.put("token", TOKEN);
params.put("__refresh_cache","1")
String content = new HttpRequest().get("https://open.fangjia.com/v2/address/standard", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
while (result == null) {
result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/v2/address/standard", params, "utf-8")
println("${city} ${address} 程序调用异常")
Thread.sleep(6000)
}
if (result.getInt("code") == 0) {
return result.getJSONObject("result");
} else if (result.getInt("code") !=0) {
println("${city} ${address}")
}
} catch (Exception e) {
println("${city} ${address} ${e.getMessage()}")
e.printStackTrace();
}
return null
}
public static JSONObject standardize(String city, String address) {
try {
Map<String, String> params = new HashMap<String, String>();
@@ -593,6 +680,90 @@ public class OpenApiClient {
}
static JSONObject facilitys(String city, String region, String name) {
Map<String, String> params = new HashMap<String, String>();
params.put("token", TOKEN);
params.put("city", city)
params.put("region", region)
params.put("name", name)
try {
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/district/facilitys", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
return result
} catch (Exception e) {
}
return null
}
static JSONObject searchPaimais(String city, String region, String name, limit) {
Map<String, String> params = new HashMap<String, String>();
params.put("token", TOKEN);
params.put("city", city)
params.put("district", region)
params.put("name", name)
params.put("limit", "10")
try {
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/auction/searchPaimais", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
return result
} catch (Exception e) {
}
return null
}
static JSONObject searchDealPrice(String id, String startDate, limit) {
Map<String, String> params = new HashMap<String, String>();
params.put("token", TOKEN);
params.put("id", id)
params.put("startDate", startDate)
params.put("limit", limit.toString())
try {
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/dealPrice/searchList", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
return result
} catch (Exception e) {
}
return null
}
static JSONObject houseList(String id, String startDate, limit) {
Map<String, String> params = new HashMap<String, String>();
params.put("token", TOKEN);
params.put("id", id)
params.put("startTime", startDate)
params.put("pageSize", limit.toString())
try {
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/house/list", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
return result
} catch (Exception e) {
}
return null
}
static JSONObject evaluaionList(String id, String startDate, limit) {
Map<String, String> params = new HashMap<String, String>();
params.put("token", TOKEN);
params.put("id", id)
params.put("startDate", startDate)
params.put("limit", limit.toString())
try {
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/evaluation/bsList", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
return result
} catch (Exception e) {
}
return null
}
static JSONObject districtDetail(String city, String region, String name) {
Map<String, String> params = new HashMap<String, String>();
params.put("token", TOKEN);
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20220301", "yyyyMMdd")
def endDate = DateUtil.parseDate("20220401", "yyyyMMdd")
def startDate = DateUtil.parseDate("20220401", "yyyyMMdd")
def endDate = DateUtil.parseDate("20220501", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
@@ -0,0 +1,11 @@
package com.fangjia.service.datastore;
import com.fangjia.dal.mongo.model.BsEvaluation;
import java.util.Date;
import java.util.List;
public interface BsEvaluationService {
List<BsEvaluation> getNearlySameEvaluation(String id, Integer minPrice, Integer maxPrice, Date startDate, Date endDate, Integer limit);
}
@@ -0,0 +1,44 @@
package com.fangjia.service.datastore;
import com.fangjia.dal.QueryParam;
import com.fangjia.dal.SortParam;
import com.fangjia.dal.mongo.dao.BsEvaluationDao;
import com.fangjia.dal.mongo.dao.EvaluationDao;
import com.fangjia.dal.mongo.model.BsEvaluation;
import com.fangjia.dal.mongo.model.bkoff.DealPrice;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
public class BsEvaluationServiceImpl implements BsEvaluationService {
private BsEvaluationDao bsEvaluationDao;
public void setBsEvaluationDao(BsEvaluationDao bsEvaluationDao) {
this.bsEvaluationDao = bsEvaluationDao;
}
@Override
public List<BsEvaluation> getNearlySameEvaluation(String id, Integer minPrice, Integer maxPrice, Date startDate, Date endDate, Integer limit) {
List<QueryParam> params = new ArrayList<QueryParam>();
params.add(new QueryParam("commid", id));
if(startDate!=null){
params.add(new QueryParam("cDate", QueryParam.GREATER_THAN_EQUAL_OPERATOR, startDate));
}
if(endDate!=null){
params.add(new QueryParam("cDate", QueryParam.LESS_THAN_EQUAL_OPERATOR, startDate));
}
if (minPrice > 0) {
params.add(new QueryParam("unitPrice", QueryParam.GREATER_THAN_OPERATOR, minPrice));
}
if (maxPrice > 0) {
params.add(new QueryParam("unitPrice", QueryParam.LESS_THAN_OPERATOR, maxPrice));
}
List<SortParam> sortParams = new ArrayList<SortParam>();
sortParams.add(new SortParam("cDate", false));
List<BsEvaluation> bsEvaluations = bsEvaluationDao.findAllByParams(params, sortParams, limit);
return bsEvaluations;
}
}
@@ -94,4 +94,6 @@ public interface HouseLoudongService {
public List<HouseLoudong> findAll(List<QueryParam> queryParams, Pager pager);
List<String> findAllSources();
public List<HouseLoudong> findAll(String id, String houseNum, String unitNum, String roomNum);
}
@@ -297,6 +297,22 @@ public class HouseLoudongServiceImpl implements HouseLoudongService {
return froms;
}
@Override
public List<HouseLoudong> findAll(String id, String houseNum, String unitNum, String roomNum) {
List<QueryParam> params = new ArrayList<QueryParam>();
params.add(new QueryParam("districtId", id));
if (StringUtils.isNotBlank(houseNum)) {
params.add(new QueryParam("houseNum", houseNum));
}
if (StringUtils.isNotBlank(unitNum)) {
params.add(new QueryParam("houseNumUnit", unitNum));
}
if (StringUtils.isNotBlank(roomNum)) {
params.add(new QueryParam("roomNum", roomNum));
}
return houseLoudongBkoffDao.findAllByParams(params);
}
@Override
public List<HouseLoudong> queryHouseLoudongList(String city, String districtName, String roomNum, Pager pager) {
List<QueryParam> params = new ArrayList<QueryParam>();
@@ -23,6 +23,8 @@ public interface LoudongService {
List<Loudong> findAll(String city, String region, String name, String houseNumber, String unitNumber, int limit);
List<Loudong> findAll(String id, String houseNumber, String unitNumber);
Map<String, XSSFWorkbook> findAll(List<QueryParam> query, String city);
Loudong findById(String id);
@@ -41,4 +43,6 @@ public interface LoudongService {
void save(Map<String, Object> data);
List<String> findAllSources();
Loudong findForEvaluate(String idInSeaweed, String houseNumber, String unitNumber);
}
@@ -148,6 +148,23 @@ public class LoudongServiceImpl implements LoudongService {
return loudongDao.findAllByParams(params,null, 0 , limit, ADDRESS_VERIFY_FIELDS);
}
@Override
public List<Loudong> findAll(String id, String houseNumber, String unitNumber) {
List<QueryParam> params = new ArrayList<QueryParam>();
if(StringUtils.isNotBlank(id)) {
params.add(new QueryParam("idInSeaweed", id));
}
if(StringUtils.isNotBlank(houseNumber)) {
params.add(new QueryParam("houseNumber", houseNumber));
}
if(StringUtils.isNotBlank(unitNumber)) {
params.add(new QueryParam("unitNumber", unitNumber));
}
return loudongDao.findAllByParams(params);
}
@Override
public Loudong findById(String id) {
return loudongDao.findById(id);
@@ -318,4 +335,23 @@ public class LoudongServiceImpl implements LoudongService {
}
return froms;
}
@Override
public Loudong findForEvaluate(String idInSeaweed, String houseNumber, String unitNumber){
if(StringUtils.isBlank(idInSeaweed) || StringUtils.isBlank(houseNumber)){
return null;
}
List<QueryParam> params = Lists.newArrayList();
params.add(new QueryParam("idInSeaweed", idInSeaweed));
params.add(new QueryParam("houseNumber", houseNumber));
if(StringUtils.isNotBlank(unitNumber)) {
params.add(new QueryParam("unitNumber", unitNumber));
}
params.add(new QueryParam("floors", QueryParam.GREATER_THAN_OPERATOR, 0));
params.add(new QueryParam("floors", QueryParam.LESS_THAN_OPERATOR, 60));
params.add(new QueryParam("status", QueryParam.NOT_EQUAL_OPERATOR, 0));
List<SortParam> sortParams = ImmutableList.of(new SortParam("floors", false));
return loudongDao.findFirstByParams(params, sortParams);
}
}
@@ -82,6 +82,8 @@ public interface DealPriceService {
public List<DealPrice> getNearlySameDealPrice(DistrictKey key, int minPrice, int maxPrice, int radius, int limit);
public List<DealPrice> getNearlySameDealPrice(String districtId, Integer minPrice, Integer maxPrice,Date startDate, Date endDate, int limit);
public List<DealPrice> getDisSameDealPrice(DistrictKey key, int minPrice, int maxPrice, int limit);
/**
@@ -354,6 +354,29 @@ public class DealPriceServiceImpl implements DealPriceService {
return houses;
}
@Override
public List<DealPrice> getNearlySameDealPrice(String districtId, Integer minPrice, Integer maxPrice, Date startDate, Date endDate, int limit) {
List<QueryParam> params = new ArrayList<QueryParam>();
params.add(new QueryParam("matchId", districtId));
if(startDate!=null){
params.add(new QueryParam("tradeDate", QueryParam.GREATER_THAN_EQUAL_OPERATOR, startDate));
}
if(endDate!=null){
params.add(new QueryParam("tradeDate", QueryParam.LESS_THAN_EQUAL_OPERATOR, startDate));
}
if (minPrice > 0) {
params.add(new QueryParam("avgPrice", QueryParam.GREATER_THAN_OPERATOR, minPrice));
}
if (maxPrice > 0) {
params.add(new QueryParam("avgPrice", QueryParam.LESS_THAN_OPERATOR, maxPrice));
}
List<SortParam> sortParams = new ArrayList<SortParam>();
sortParams.add(new SortParam("tradeDate", false));
List<DealPrice> dealPrices = dealPriceDao.findAllByParams(params, sortParams, limit);
return dealPrices;
}
@Override
public List<DealPrice> getDisSameDealPrice(DistrictKey key, int minPrice, int maxPrice, int limit) {
List<QueryParam> params = new ArrayList<QueryParam>();