Merge remote-tracking branch 'origin/master'
This commit is contained in:
@@ -0,0 +1,9 @@
|
||||
package com.fangjia.dal.mongo.dao.shop;
|
||||
|
||||
import com.fangjia.dal.MongoDao;
|
||||
import com.fangjia.dal.mongo.model.ShopRentStat;
|
||||
|
||||
public interface ShopRentStatDao extends MongoDao<ShopRentStat> {
|
||||
|
||||
|
||||
}
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
package com.fangjia.dal.mongo.dao.shop;
|
||||
|
||||
import com.fangjia.dal.MongoDaoImpl;
|
||||
import com.fangjia.dal.mongo.model.ShopRentStat;
|
||||
|
||||
public class ShopRentStatDaoImpl extends MongoDaoImpl<ShopRentStat> implements ShopRentStatDao {
|
||||
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package com.fangjia.dal.mongo.dao.shop;
|
||||
|
||||
import com.fangjia.dal.MongoDao;
|
||||
import com.fangjia.dal.mongo.model.ShopSellStat;
|
||||
|
||||
public interface ShopSellStatDao extends MongoDao<ShopSellStat> {
|
||||
|
||||
|
||||
}
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
package com.fangjia.dal.mongo.dao.shop;
|
||||
|
||||
import com.fangjia.dal.MongoDaoImpl;
|
||||
import com.fangjia.dal.mongo.model.ShopSellStat;
|
||||
|
||||
public class ShopSellStatDaoImpl extends MongoDaoImpl<ShopSellStat> implements ShopSellStatDao {
|
||||
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package com.fangjia.dal.mongo.model;
|
||||
|
||||
import com.fangjia.dal.MongoEntity;
|
||||
|
||||
import javax.persistence.*;
|
||||
|
||||
@Entity
|
||||
@Table(name = "fangjia_stat.shop_rent_stat")
|
||||
public class ShopRentStat implements MongoEntity {
|
||||
private static final long serialVersionUID = 4425229045088250573L;
|
||||
|
||||
private String id;
|
||||
private String cat;
|
||||
private String city;
|
||||
private String district;
|
||||
private Integer date;
|
||||
private Double avgPrice;
|
||||
private String name;
|
||||
|
||||
@Id
|
||||
@GeneratedValue
|
||||
@Column(name = "_id", unique = true, nullable = false)
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public void setId(String id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
@Column(name = "cat")
|
||||
public String getCat() {
|
||||
return cat;
|
||||
}
|
||||
|
||||
public void setCat(String cat) {
|
||||
this.cat = cat;
|
||||
}
|
||||
|
||||
@Column(name = "city")
|
||||
public String getCity() {
|
||||
return city;
|
||||
}
|
||||
|
||||
public void setCity(String city) {
|
||||
this.city = city;
|
||||
}
|
||||
|
||||
@Column(name = "district")
|
||||
public String getDistrict() {
|
||||
return district;
|
||||
}
|
||||
|
||||
public void setDistrict(String district) {
|
||||
this.district = district;
|
||||
}
|
||||
|
||||
@Column(name = "date")
|
||||
public Integer getDate() {
|
||||
return date;
|
||||
}
|
||||
|
||||
public void setDate(Integer date) {
|
||||
this.date = date;
|
||||
}
|
||||
|
||||
@Column(name = "avg_price")
|
||||
public Double getAvgPrice() {
|
||||
return avgPrice;
|
||||
}
|
||||
|
||||
public void setAvgPrice(Double avgPrice) {
|
||||
this.avgPrice = avgPrice;
|
||||
}
|
||||
|
||||
@Column(name = "name")
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package com.fangjia.dal.mongo.model;
|
||||
|
||||
import com.fangjia.dal.MongoEntity;
|
||||
|
||||
import javax.persistence.*;
|
||||
|
||||
@Entity
|
||||
@Table(name = "fangjia_stat.shop_sell_stat")
|
||||
public class ShopSellStat implements MongoEntity {
|
||||
private static final long serialVersionUID = 4425229045088250573L;
|
||||
|
||||
private String id;
|
||||
private String cat;
|
||||
private String city;
|
||||
private String district;
|
||||
private Integer date;
|
||||
private Integer avgPrice;
|
||||
private String name;
|
||||
|
||||
@Id
|
||||
@GeneratedValue
|
||||
@Column(name = "_id", unique = true, nullable = false)
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public void setId(String id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
@Column(name = "cat")
|
||||
public String getCat() {
|
||||
return cat;
|
||||
}
|
||||
|
||||
public void setCat(String cat) {
|
||||
this.cat = cat;
|
||||
}
|
||||
|
||||
@Column(name = "city")
|
||||
public String getCity() {
|
||||
return city;
|
||||
}
|
||||
|
||||
public void setCity(String city) {
|
||||
this.city = city;
|
||||
}
|
||||
|
||||
@Column(name = "district")
|
||||
public String getDistrict() {
|
||||
return district;
|
||||
}
|
||||
|
||||
public void setDistrict(String district) {
|
||||
this.district = district;
|
||||
}
|
||||
|
||||
@Column(name = "date")
|
||||
public Integer getDate() {
|
||||
return date;
|
||||
}
|
||||
|
||||
public void setDate(Integer date) {
|
||||
this.date = date;
|
||||
}
|
||||
|
||||
@Column(name = "avg_price")
|
||||
public Integer getAvgPrice() {
|
||||
return avgPrice;
|
||||
}
|
||||
|
||||
public void setAvgPrice(Integer avgPrice) {
|
||||
this.avgPrice = avgPrice;
|
||||
}
|
||||
|
||||
@Column(name = "name")
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,614 @@
|
||||
package com.fangjia.open.controllers;
|
||||
|
||||
import com.fangjia.core.RegionData;
|
||||
import com.fangjia.core.util.AnalyticsUtil;
|
||||
import com.fangjia.core.util.DataUtil;
|
||||
import com.fangjia.core.util.HttpRequest;
|
||||
import com.fangjia.core.util.LoggerFactory;
|
||||
import com.fangjia.dal.common.ServerAreaData;
|
||||
import com.fangjia.dal.mongo.fangjia.model.ServerArea;
|
||||
import com.fangjia.dal.mongo.model.Evaluation;
|
||||
import com.fangjia.dal.mongo.model.ShopRentStat;
|
||||
import com.fangjia.open.HttpCode;
|
||||
import com.fangjia.open.OpenHttpCodeEnum;
|
||||
import com.fangjia.open.interceptors.AuthenticationBean;
|
||||
import com.fangjia.open.interceptors.Authority;
|
||||
import com.fangjia.open.interceptors.Role;
|
||||
import com.fangjia.open.model.BasicModel;
|
||||
import com.fangjia.open.model.PropertyEvaluationModel;
|
||||
import com.fangjia.open.model.ReportApplyModel;
|
||||
import com.fangjia.open.model.RestfulModelAndView;
|
||||
import com.fangjia.open.util.CommonUtils;
|
||||
import com.fangjia.open.util.FangJiaTongUrl;
|
||||
import com.fangjia.service.address.AddressParser;
|
||||
import com.fangjia.service.datastore.FieldFilterService;
|
||||
import com.fangjia.service.datastore.AiShopService;
|
||||
import com.fangjia.service.datastore.OpenApiLogService;
|
||||
import com.fangjia.service.datastore.dto.ShopEvaluationDto;
|
||||
import net.sf.json.JSONObject;
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
import org.apache.commons.lang.ArrayUtils;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.sothis.web.mvc.ActionContext;
|
||||
import org.sothis.web.mvc.annotation.Param;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static com.fangjia.open.util.CommonUtils.convertToString;
|
||||
import static com.fangjia.open.util.EvaluationFactorUtil.*;
|
||||
|
||||
/**
|
||||
* 商铺出租服务类
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2025/4/10
|
||||
*/
|
||||
@Authority(role = Role.USER)
|
||||
public class ShopController {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(PropertyController.class);
|
||||
|
||||
private AiShopService aiShopService;
|
||||
|
||||
private OpenApiLogService openApiLogService;
|
||||
|
||||
private AddressParser addressParser;
|
||||
private FieldFilterService fieldFilterService;
|
||||
|
||||
/**
|
||||
* 城市出租均价
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public Object cityRentAction(@Param(name = "city") String city, @Param(name = "startDate") Integer startDate, @Param(name = "endDate") Integer endDate) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
if (StringUtils.isBlank(city)) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("city不能为null"));
|
||||
return mav;
|
||||
}
|
||||
if (!RegionData.isCity(city)) {
|
||||
String formatCity = RegionData.formatCity(city);
|
||||
if (StringUtils.isNotBlank(formatCity)) {
|
||||
city = formatCity;
|
||||
} else {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("city非法"));
|
||||
return mav;
|
||||
}
|
||||
}
|
||||
ServerArea serverArea = ServerAreaData.getServerAreaByCity(city);
|
||||
if (null == serverArea) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("city非法"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
if (startDate == null || endDate == null) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("startDate或endDate不能为null"));
|
||||
return mav;
|
||||
}
|
||||
if (startDate > endDate) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("startDate不能大于endDate"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
List<ShopRentStat> regionStats = aiShopService.findShopRentStat(city, null, null, "city", startDate, endDate);
|
||||
|
||||
if (CollectionUtils.isEmpty(regionStats)) {
|
||||
mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "该城市没有均价数据"));
|
||||
return mav;
|
||||
}
|
||||
mav.setModel(BasicModel.buildOKModel().setResult(regionStats));
|
||||
return mav;
|
||||
}
|
||||
|
||||
/**
|
||||
* 区域出租均价
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public Object districtRentAction(@Param(name = "city") String city, @Param(name = "district") String district,
|
||||
@Param(name = "startDate") Integer startDate, @Param(name = "endDate") Integer endDate) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
if (StringUtils.isBlank(city)) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("city不能为null"));
|
||||
return mav;
|
||||
}
|
||||
if (StringUtils.isBlank(district)) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("district不能为null"));
|
||||
return mav;
|
||||
}
|
||||
if (!RegionData.isCity(city)) {
|
||||
String formatCity = RegionData.formatCity(city);
|
||||
if (StringUtils.isNotBlank(formatCity)) {
|
||||
city = formatCity;
|
||||
} else {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("city非法"));
|
||||
return mav;
|
||||
}
|
||||
}
|
||||
ServerArea serverArea = ServerAreaData.getServerAreaByCity(city);
|
||||
if (null == serverArea) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("city非法"));
|
||||
return mav;
|
||||
}
|
||||
String region = RegionData.formatRegion(city, district);
|
||||
if (!RegionData.isRegion(city, region)) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("没有找你输入的区域"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
if (startDate == null || endDate == null) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("startDate或endDate不能为null"));
|
||||
return mav;
|
||||
}
|
||||
if (startDate > endDate) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("startDate不能大于endDate"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
List<ShopRentStat> regionStats = aiShopService.findShopRentStat(city, region, null, "district", startDate, endDate);
|
||||
|
||||
if (CollectionUtils.isEmpty(regionStats)) {
|
||||
mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "该区域没有均价数据"));
|
||||
return mav;
|
||||
}
|
||||
mav.setModel(BasicModel.buildOKModel().setResult(regionStats));
|
||||
return mav;
|
||||
}
|
||||
|
||||
/**
|
||||
* 写字楼均价走势
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public Object shopRentAction(@Param(name = "city") String city, @Param(name = "district") String district,
|
||||
@Param(name = "name") String name, @Param(name = "startDate") Integer startDate,
|
||||
@Param(name = "endDate") Integer endDate) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
if (StringUtils.isBlank(city)) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("city不能为null"));
|
||||
return mav;
|
||||
}
|
||||
if (StringUtils.isBlank(district)) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("district不能为null"));
|
||||
return mav;
|
||||
}
|
||||
if (!RegionData.isCity(city)) {
|
||||
String formatCity = RegionData.formatCity(city);
|
||||
if (StringUtils.isNotBlank(formatCity)) {
|
||||
city = formatCity;
|
||||
} else {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("city非法"));
|
||||
return mav;
|
||||
}
|
||||
}
|
||||
ServerArea serverArea = ServerAreaData.getServerAreaByCity(city);
|
||||
if (null == serverArea) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("city非法"));
|
||||
return mav;
|
||||
}
|
||||
String region = RegionData.formatRegion(city, district);
|
||||
if (!RegionData.isRegion(city, region)) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("没有找你输入的区域"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
if (startDate == null || endDate == null) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("startDate或endDate不能为null"));
|
||||
return mav;
|
||||
}
|
||||
if (startDate > endDate) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("startDate不能大于endDate"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
List<ShopRentStat> regionStats = aiShopService.findShopRentStat(city, region, name, "shop", startDate, endDate);
|
||||
|
||||
if (CollectionUtils.isEmpty(regionStats)) {
|
||||
mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "该写字楼没有均价数据"));
|
||||
return mav;
|
||||
}
|
||||
mav.setModel(BasicModel.buildOKModel().setResult(regionStats));
|
||||
return mav;
|
||||
}
|
||||
|
||||
/**
|
||||
* 写字楼楼租金地址估值
|
||||
*
|
||||
* @param model
|
||||
* @param request
|
||||
* @return
|
||||
*/
|
||||
public Object addressEvaluateRentAction(PropertyEvaluationModel model, HttpServletRequest request) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
if (StringUtils.isBlank(model.getCity()) || StringUtils.isBlank(model.getAddress()) || model.getSize() <= 0) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("参数非法:城市,地址,面积(size)为必填项"));
|
||||
return mav;
|
||||
}
|
||||
if (model.getCity().endsWith("市")) {
|
||||
model.setCity(model.getCity().replaceAll("市$", ""));
|
||||
}
|
||||
|
||||
if (model.getCarportPrice() != null && (model.getCarportPrice() < 0 || model.getCarportPrice() > 1000000)) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("carportPrice(车位)取值不合法"));
|
||||
return mav;
|
||||
}
|
||||
if (model.getMaxFloor() != null && (model.getMaxFloor() <= 0 || model.getMaxFloor() > 59)) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("maxFloor取值非法,正确范围[1,60)"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
if (model.getFloor() != null && (model.getFloor() <= 0 || model.getFloor() > 59)) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("floor取值非法,正确范围[1,60)"));
|
||||
return mav;
|
||||
}
|
||||
if (model.getMaxFloor() != null && model.getFloor() != null && model.getFloor() > model.getMaxFloor()) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("所在楼层不能大于总楼层"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
ShopEvaluationDto evaluation = aiShopService.addressEvaluateRent(buildEvaluation(model, request));
|
||||
|
||||
if (evaluation.getUnitPrice() != null && evaluation.getUnitPrice() != 0) {
|
||||
Map<String, Object> districtPriceMap = new HashMap<String, Object>();
|
||||
districtPriceMap.put("unitPrice", evaluation.getUnitPrice());
|
||||
districtPriceMap.put("city", evaluation.getCity());
|
||||
districtPriceMap.put("district", evaluation.getDistrict());
|
||||
districtPriceMap.put("name", evaluation.getName());
|
||||
districtPriceMap.put("samples", evaluation.getSeawater());
|
||||
districtPriceMap.put("location", evaluation.getLocation());
|
||||
|
||||
mav.setModel(BasicModel.buildOKModel().setResult(districtPriceMap));
|
||||
try {
|
||||
if (StringUtils.isNotBlank(evaluation.getFromUserId()) && !evaluation.getFromUserId().equals("1007497")) {
|
||||
openApiLogService.saveLog(CommonUtils.convertToMap(model), districtPriceMap, evaluation, "/shop/addressEvaluateRent");
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
return mav;
|
||||
} else {
|
||||
mav.setModel(BasicModel.buildOKModel().setMsg("暂时不支持该地址写字楼的评估").markNoData());
|
||||
return mav;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 申请估值报告
|
||||
*
|
||||
* @param model 申请报告请求参数
|
||||
* @return
|
||||
*/
|
||||
public Object assessPriceReportApplyAction(ReportApplyModel model) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
if (StringUtils.isBlank(model.getClient())) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("参数非法:委托机构为必填项"));
|
||||
return mav;
|
||||
}
|
||||
if (StringUtils.isBlank(model.getSubjectMatter())) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("参数非法:标的物为必填项"));
|
||||
return mav;
|
||||
}
|
||||
if (StringUtils.isBlank(model.getCity()) || model.getCity().contains("全国")) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("参数非法:城市为必填项且不能为全国"));
|
||||
return mav;
|
||||
}
|
||||
if (StringUtils.isBlank(model.getDistrict()) || model.getDistrict().contains("全国")) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("参数非法:区域为必填项且不能为全国"));
|
||||
return mav;
|
||||
}
|
||||
if (model.getSize() == null || model.getSize() <= 0 ) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("参数非法:面积(size)为必填项且大于0"));
|
||||
return mav;
|
||||
}
|
||||
if (model.getCity().endsWith("市")) {
|
||||
model.setCity(model.getCity().replaceAll("市$", ""));
|
||||
}
|
||||
if (model.getDistrict().endsWith("区")) {
|
||||
model.setDistrict(model.getDistrict().replaceAll("区$", ""));
|
||||
}
|
||||
if (model.getMaxFloor() != null && (model.getMaxFloor() <= 0 || model.getMaxFloor() > 59)) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("maxFloor取值非法,正确范围[1,60)"));
|
||||
return mav;
|
||||
}
|
||||
if (model.getFloor() != null && (model.getFloor() <= 0 || model.getFloor() > 59)) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("floor取值非法,正确范围[1,60)"));
|
||||
return mav;
|
||||
}
|
||||
if (model.getMaxFloor() != null && model.getFloor() != null && model.getFloor() > model.getMaxFloor()) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("所在楼层不能大于总楼层"));
|
||||
return mav;
|
||||
}
|
||||
HttpRequest request = new HttpRequest();
|
||||
JSONObject json = null;
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("client", model.getClient());
|
||||
params.put("subjectMatter", model.getSubjectMatter());
|
||||
params.put("city", model.getCity());
|
||||
params.put("address", StringUtils.isNotBlank(model.getAddress()) ? model.getAddress() : null);
|
||||
params.put("district", model.getDistrict());
|
||||
params.put("name", StringUtils.isNotBlank(model.getName()) ? model.getName() : null);
|
||||
params.put("houseNum", StringUtils.isNotBlank(model.getBuildingNumber()) ? model.getBuildingNumber() : null);
|
||||
params.put("roomNum", StringUtils.isNotBlank(model.getUnitNumber()) ? model.getUnitNumber() : null);
|
||||
params.put("size", String.valueOf(model.getSize()));
|
||||
params.put("houseFloor", model.getFloor() != null ? String.valueOf(model.getFloor()) : null);
|
||||
params.put("houseHeight", model.getMaxFloor() != null ? String.valueOf(model.getMaxFloor()) : null);
|
||||
params.put("buildType", StringUtils.isNotBlank(model.getPropertyType()) ? model.getPropertyType() : "0");
|
||||
params.put("direction", StringUtils.isNotBlank(model.getDirection()) ? model.getDirection() : "3");
|
||||
params.put("fitment", StringUtils.isNotBlank(model.getDecoration()) ? model.getDecoration() : "2");
|
||||
JSONObject jsonObject = JSONObject.fromObject(params);
|
||||
Map<String, String> headers = new HashMap<String, String>();
|
||||
headers.put("Authorization", "Bearer " + FangJiaTongUrl.FANG_JIA_TONG_TOKEN);
|
||||
headers.put("Content-Type", "application/json");
|
||||
try {
|
||||
String ret = request.postWithBody(FangJiaTongUrl.FANG_JIA_TONG_URL_UAT + FangJiaTongUrl.ASSESS_PRICE_REPORT_APPLY, jsonObject, headers, "UTF-8");
|
||||
json = JSONObject.fromObject(FangJiaTongUrl.replaceAll(ret));
|
||||
} catch (Exception e) {
|
||||
try {
|
||||
String ret = request.postWithBody(FangJiaTongUrl.FANG_JIA_TONG_URL_UAT + FangJiaTongUrl.ASSESS_PRICE_REPORT_APPLY, jsonObject, headers, "UTF-8");
|
||||
json = JSONObject.fromObject(FangJiaTongUrl.replaceAll(ret));
|
||||
} catch (Exception ex) {
|
||||
ex.printStackTrace();
|
||||
}
|
||||
}
|
||||
if (json != null) {
|
||||
if (json.getInt("code") == 200 && json.containsKey("data")) {
|
||||
HashMap<String, String> result = new HashMap<String, String>();
|
||||
result.put("caseId", String.valueOf(json.get("data")));
|
||||
mav.setModel(BasicModel.buildOKModel().setResult(result));
|
||||
} else {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("申请估值报告失败:" + json.getString("message")));
|
||||
}
|
||||
} else {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("申请估值报告失败:调用申请报告接口返回为空"));
|
||||
}
|
||||
return mav;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取估值报告
|
||||
*
|
||||
* @param caseId 估值报告ID
|
||||
* @return
|
||||
*/
|
||||
public Object assessPriceReportFetchAction(@Param(name = "caseId") String caseId) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
if (StringUtils.isBlank(caseId)) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("参数非法:估值报告业务编号为必填项"));
|
||||
return mav;
|
||||
}
|
||||
HttpRequest request = new HttpRequest();
|
||||
JSONObject json = null;
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("caseId", caseId);
|
||||
JSONObject jsonObject = JSONObject.fromObject(params);
|
||||
Map<String, String> headers = new HashMap<String, String>();
|
||||
headers.put("Authorization", "Bearer " + FangJiaTongUrl.FANG_JIA_TONG_TOKEN);
|
||||
headers.put("Content-Type", "application/json");
|
||||
try {
|
||||
String ret = request.postWithBody(FangJiaTongUrl.FANG_JIA_TONG_URL_UAT + FangJiaTongUrl.ASSESS_PRICE_REPORT_FETCH, jsonObject, headers, "UTF-8");
|
||||
json = JSONObject.fromObject(FangJiaTongUrl.replaceAll(ret));
|
||||
} catch (Exception e) {
|
||||
try {
|
||||
String ret = request.postWithBody(FangJiaTongUrl.FANG_JIA_TONG_URL_UAT + FangJiaTongUrl.ASSESS_PRICE_REPORT_FETCH, jsonObject, headers, "UTF-8");
|
||||
json = JSONObject.fromObject(FangJiaTongUrl.replaceAll(ret));
|
||||
} catch (Exception ex) {
|
||||
ex.printStackTrace();
|
||||
}
|
||||
}
|
||||
if (json != null) {
|
||||
if (json.getInt("code") == 200 && json.containsKey("data")) {
|
||||
JSONObject resultJson = json.getJSONObject("data");
|
||||
mav.setModel(BasicModel.buildOKModel().setResult(resultJson));
|
||||
} else {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("获取估值报告失败:" + json.getString("message")));
|
||||
}
|
||||
} else {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("申请报告失败:调用获取估值报告接口返回为空"));
|
||||
}
|
||||
return mav;
|
||||
}
|
||||
|
||||
private Integer getFloor(String roomNumber) {
|
||||
if (StringUtils.isNotBlank(roomNumber) && DataUtil.isNumber(roomNumber)) {
|
||||
if (roomNumber.length() == 3) {
|
||||
return Integer.valueOf(roomNumber.substring(0, 1));
|
||||
} else if (roomNumber.length() == 4) {
|
||||
return Integer.valueOf(roomNumber.substring(0, 2));
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 通过EvaluationModel组装Evaluation实体
|
||||
*
|
||||
* @param model
|
||||
* @param request
|
||||
* @return
|
||||
*/
|
||||
private Evaluation buildEvaluation(PropertyEvaluationModel model, HttpServletRequest request) {
|
||||
Evaluation evaluation = new Evaluation();
|
||||
evaluation.setLevel(model.getLevel());
|
||||
evaluation.setFitment(model.getFitment());
|
||||
evaluation.setType(model.getType());
|
||||
|
||||
evaluation.setCity(model.getCity());
|
||||
evaluation.setRegion(model.getRegion());
|
||||
evaluation.setDistrictName(model.getName());
|
||||
evaluation.setAddress(model.getAddress());
|
||||
evaluation.setDirection(model.getDirection());
|
||||
evaluation.setNoise(NOISE.get(model.getNoise()));
|
||||
evaluation.setType(model.getType());
|
||||
evaluation.setSight(SIGHT.get(model.getView()));
|
||||
evaluation.setAtticArea(model.getAtticSize());
|
||||
evaluation.setTerraceArea(model.getOuterSize());
|
||||
evaluation.setBasementArea(model.getBasementSize());
|
||||
evaluation.setGardenArea(model.getGardenSize());
|
||||
String worryFacility = model.getNegativeFactor();
|
||||
evaluation.setWorryFacility(NEGATIVE_FACTOR.get(worryFacility));
|
||||
evaluation.setCreationDate(new Date());
|
||||
evaluation.setModificationDate(new Date());
|
||||
evaluation.setClientNo(model.getClientNo());
|
||||
if (0 != model.getAge()) {
|
||||
evaluation.setAge(model.getAge());
|
||||
}
|
||||
evaluation.setArea(model.getSize());
|
||||
if (ArrayUtils.isNotEmpty(model.getBuildQuality())) {
|
||||
evaluation.setBuildQuality(convertToString(model.getBuildQuality()));
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getBuildType())) {
|
||||
evaluation.setBuildType(model.getBuildType());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getCarport())) {
|
||||
evaluation.setCarport(model.getCarport());
|
||||
}
|
||||
if (model.getDecorationAge() >= 1 && model.getDecorationAge() <= 5) {
|
||||
evaluation.setFitmentAge(model.getDecorationAge());
|
||||
}
|
||||
evaluation.setCarportPrice(model.getCarportPrice());
|
||||
evaluation.setCity(model.getCity());
|
||||
evaluation.setDistrictName(model.getName());
|
||||
if (model.getDecorationCost() != null && model.getDecorationCost() > 0) {
|
||||
evaluation.setFitmentCharge(model.getDecorationCost());
|
||||
}
|
||||
if (0 != model.getHall()) {
|
||||
evaluation.setHall(model.getHall());
|
||||
}
|
||||
if (null != model.getFloor()) {
|
||||
evaluation.setFloor(model.getFloor());
|
||||
}
|
||||
if (model.getMaxFloor() != null && 0 != model.getMaxFloor()) {
|
||||
evaluation.setHeight(model.getMaxFloor());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getBuildingNumber())) {
|
||||
evaluation.setHouseNum(model.getBuildingNumber());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getHouseProperty())) {
|
||||
evaluation.setHouseProperty(model.getHouseProperty());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getInsidepos())) {
|
||||
evaluation.setInsidepos(model.getInsidepos());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getLightingVentilation())) {
|
||||
evaluation.setLightingVentilation(model.getLightingVentilation());
|
||||
}
|
||||
|
||||
if (StringUtils.isNotBlank(model.getOtherFactor())) {
|
||||
evaluation.setOtherFactor(model.getOtherFactor());
|
||||
}
|
||||
evaluation.setRegion(model.getDistrict());
|
||||
if (null != model.getRoom()) {
|
||||
evaluation.setRoom(model.getRoom());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getRoomNumber())) {
|
||||
evaluation.setRoomNum(model.getRoomNumber());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getSideset())) {
|
||||
evaluation.setSideset(model.getSideset());
|
||||
}
|
||||
if (0 != model.getToilet()) {
|
||||
evaluation.setToilet(model.getToilet());
|
||||
}
|
||||
if (model.getFacilities() != null) {
|
||||
String facilities = model.getFacilities();
|
||||
String[] fs = facilities.split(",");
|
||||
for (int i = 0; i < fs.length; i++) {
|
||||
fs[i] = FACILITIES.get(fs[i]);
|
||||
}
|
||||
evaluation.setFacilities(convertToString(fs));
|
||||
}
|
||||
|
||||
if (StringUtils.isNotBlank(model.getFacade())) {
|
||||
evaluation.setFacade(model.getFacade());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getTraffice())) {
|
||||
evaluation.setTraffice(model.getTraffice());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getShape())) {
|
||||
evaluation.setShape(model.getShape());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getObstacle())) {
|
||||
evaluation.setObstacle(model.getObstacle());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getIndustry())) {
|
||||
evaluation.setIndustry(model.getIndustry());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getSupportDevice())) {
|
||||
evaluation.setSupportDevice(model.getSupportDevice());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getLeftPosition())) {
|
||||
evaluation.setLeftPosition(model.getLeftPosition());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getShopType())) {
|
||||
evaluation.setShopType(model.getShopType());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getShopUnique())) {
|
||||
evaluation.setShopUnique(model.getShopUnique());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getLocation())) {
|
||||
evaluation.setLocation(model.getLocation());
|
||||
}
|
||||
if (model.isNewDistrict()) {
|
||||
evaluation.setNewDistrict(model.isNewDistrict());
|
||||
}
|
||||
String floorDesc = "中";
|
||||
if (evaluation.getFloor() != null && evaluation.getHeight() != null) {
|
||||
Double floorInfo = evaluation.getFloor() * 1.0 / evaluation.getHeight();
|
||||
if (floorInfo <= 0.33) {
|
||||
floorDesc = "低";
|
||||
} else if (floorInfo > 0.66) {
|
||||
floorDesc = "高";
|
||||
}
|
||||
evaluation.setFloorDesc(floorDesc);
|
||||
}
|
||||
String ip = AnalyticsUtil.getIpAddress(ActionContext.getContext().getRequest());
|
||||
evaluation.setIp(ip);
|
||||
AuthenticationBean authenticationBean = (AuthenticationBean) request
|
||||
.getAttribute(AuthenticationBean.ATTR_AUTHENTICATION_BEAN);
|
||||
if (authenticationBean != null && authenticationBean.apiUser != null) {
|
||||
evaluation.setFromUserId(String.valueOf(authenticationBean.apiUser.getId()));
|
||||
}
|
||||
evaluation.setSource("api_shop_evaluateRent");
|
||||
return evaluation;
|
||||
}
|
||||
|
||||
private static final Map<String, String> FACILITIES = new HashMap<String, String>() {
|
||||
{
|
||||
put("0", "健身房");
|
||||
put("1", "游泳池");
|
||||
put("2", "会所");
|
||||
put("3", "网球场");
|
||||
put("4", "球场");
|
||||
put("5", "棋牌室");
|
||||
put("6", "停车场");
|
||||
put("7", "图书馆");
|
||||
put("8", "活动中心");
|
||||
}
|
||||
};
|
||||
|
||||
public void setOpenApiLogService(OpenApiLogService openApiLogService) {
|
||||
this.openApiLogService = openApiLogService;
|
||||
}
|
||||
|
||||
public void setFieldFilterService(FieldFilterService fieldFilterService) {
|
||||
this.fieldFilterService = fieldFilterService;
|
||||
}
|
||||
|
||||
public void setAiShopService(AiShopService aiShopService) {
|
||||
this.aiShopService = aiShopService;
|
||||
}
|
||||
}
|
||||
+614
@@ -0,0 +1,614 @@
|
||||
package com.fangjia.open.controllers;
|
||||
|
||||
import com.fangjia.core.RegionData;
|
||||
import com.fangjia.core.util.AnalyticsUtil;
|
||||
import com.fangjia.core.util.DataUtil;
|
||||
import com.fangjia.core.util.HttpRequest;
|
||||
import com.fangjia.core.util.LoggerFactory;
|
||||
import com.fangjia.dal.common.ServerAreaData;
|
||||
import com.fangjia.dal.mongo.fangjia.model.ServerArea;
|
||||
import com.fangjia.dal.mongo.model.Evaluation;
|
||||
import com.fangjia.dal.mongo.model.ShopSellStat;
|
||||
import com.fangjia.open.HttpCode;
|
||||
import com.fangjia.open.OpenHttpCodeEnum;
|
||||
import com.fangjia.open.interceptors.AuthenticationBean;
|
||||
import com.fangjia.open.interceptors.Authority;
|
||||
import com.fangjia.open.interceptors.Role;
|
||||
import com.fangjia.open.model.BasicModel;
|
||||
import com.fangjia.open.model.PropertyEvaluationModel;
|
||||
import com.fangjia.open.model.ReportApplyModel;
|
||||
import com.fangjia.open.model.RestfulModelAndView;
|
||||
import com.fangjia.open.util.CommonUtils;
|
||||
import com.fangjia.open.util.FangJiaTongUrl;
|
||||
import com.fangjia.service.address.AddressParser;
|
||||
import com.fangjia.service.datastore.AiShopService;
|
||||
import com.fangjia.service.datastore.FieldFilterService;
|
||||
import com.fangjia.service.datastore.OpenApiLogService;
|
||||
import com.fangjia.service.datastore.dto.ShopEvaluationDto;
|
||||
import net.sf.json.JSONObject;
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
import org.apache.commons.lang.ArrayUtils;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.sothis.web.mvc.ActionContext;
|
||||
import org.sothis.web.mvc.annotation.Param;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static com.fangjia.open.util.CommonUtils.convertToString;
|
||||
import static com.fangjia.open.util.EvaluationFactorUtil.*;
|
||||
|
||||
/**
|
||||
* 商铺出售服务类
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2025/4/14
|
||||
*/
|
||||
@Authority(role = Role.USER)
|
||||
public class ShopSellController {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(PropertyController.class);
|
||||
|
||||
private AiShopService aiShopService;
|
||||
|
||||
private OpenApiLogService openApiLogService;
|
||||
|
||||
private AddressParser addressParser;
|
||||
private FieldFilterService fieldFilterService;
|
||||
|
||||
/**
|
||||
* 城市出租均价
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public Object citySellAction(@Param(name = "city") String city, @Param(name = "startDate") Integer startDate, @Param(name = "endDate") Integer endDate) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
if (StringUtils.isBlank(city)) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("city不能为null"));
|
||||
return mav;
|
||||
}
|
||||
if (!RegionData.isCity(city)) {
|
||||
String formatCity = RegionData.formatCity(city);
|
||||
if (StringUtils.isNotBlank(formatCity)) {
|
||||
city = formatCity;
|
||||
} else {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("city非法"));
|
||||
return mav;
|
||||
}
|
||||
}
|
||||
ServerArea serverArea = ServerAreaData.getServerAreaByCity(city);
|
||||
if (null == serverArea) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("city非法"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
if (startDate == null || endDate == null) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("startDate或endDate不能为null"));
|
||||
return mav;
|
||||
}
|
||||
if (startDate > endDate) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("startDate不能大于endDate"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
List<ShopSellStat> regionStats = aiShopService.findShopSellStat(city, null, null, "city", startDate, endDate);
|
||||
|
||||
if (CollectionUtils.isEmpty(regionStats)) {
|
||||
mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "该城市没有均价数据"));
|
||||
return mav;
|
||||
}
|
||||
mav.setModel(BasicModel.buildOKModel().setResult(regionStats));
|
||||
return mav;
|
||||
}
|
||||
|
||||
/**
|
||||
* 区域出租均价
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public Object districtSellAction(@Param(name = "city") String city, @Param(name = "district") String district,
|
||||
@Param(name = "startDate") Integer startDate, @Param(name = "endDate") Integer endDate) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
if (StringUtils.isBlank(city)) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("city不能为null"));
|
||||
return mav;
|
||||
}
|
||||
if (StringUtils.isBlank(district)) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("district不能为null"));
|
||||
return mav;
|
||||
}
|
||||
if (!RegionData.isCity(city)) {
|
||||
String formatCity = RegionData.formatCity(city);
|
||||
if (StringUtils.isNotBlank(formatCity)) {
|
||||
city = formatCity;
|
||||
} else {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("city非法"));
|
||||
return mav;
|
||||
}
|
||||
}
|
||||
ServerArea serverArea = ServerAreaData.getServerAreaByCity(city);
|
||||
if (null == serverArea) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("city非法"));
|
||||
return mav;
|
||||
}
|
||||
String region = RegionData.formatRegion(city, district);
|
||||
if (!RegionData.isRegion(city, region)) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("没有找你输入的区域"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
if (startDate == null || endDate == null) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("startDate或endDate不能为null"));
|
||||
return mav;
|
||||
}
|
||||
if (startDate > endDate) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("startDate不能大于endDate"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
List<ShopSellStat> regionStats = aiShopService.findShopSellStat(city, region, null, "district", startDate, endDate);
|
||||
|
||||
if (CollectionUtils.isEmpty(regionStats)) {
|
||||
mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "该区域没有均价数据"));
|
||||
return mav;
|
||||
}
|
||||
mav.setModel(BasicModel.buildOKModel().setResult(regionStats));
|
||||
return mav;
|
||||
}
|
||||
|
||||
/**
|
||||
* 写字楼均价走势
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public Object shopSellAction(@Param(name = "city") String city, @Param(name = "district") String district,
|
||||
@Param(name = "name") String name, @Param(name = "startDate") Integer startDate,
|
||||
@Param(name = "endDate") Integer endDate) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
if (StringUtils.isBlank(city)) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("city不能为null"));
|
||||
return mav;
|
||||
}
|
||||
if (StringUtils.isBlank(district)) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("district不能为null"));
|
||||
return mav;
|
||||
}
|
||||
if (!RegionData.isCity(city)) {
|
||||
String formatCity = RegionData.formatCity(city);
|
||||
if (StringUtils.isNotBlank(formatCity)) {
|
||||
city = formatCity;
|
||||
} else {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("city非法"));
|
||||
return mav;
|
||||
}
|
||||
}
|
||||
ServerArea serverArea = ServerAreaData.getServerAreaByCity(city);
|
||||
if (null == serverArea) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("city非法"));
|
||||
return mav;
|
||||
}
|
||||
String region = RegionData.formatRegion(city, district);
|
||||
if (!RegionData.isRegion(city, region)) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("没有找你输入的区域"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
if (startDate == null || endDate == null) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("startDate或endDate不能为null"));
|
||||
return mav;
|
||||
}
|
||||
if (startDate > endDate) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("startDate不能大于endDate"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
List<ShopSellStat> regionStats = aiShopService.findShopSellStat(city, region, name, "shop", startDate, endDate);
|
||||
|
||||
if (CollectionUtils.isEmpty(regionStats)) {
|
||||
mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "该写字楼没有均价数据"));
|
||||
return mav;
|
||||
}
|
||||
mav.setModel(BasicModel.buildOKModel().setResult(regionStats));
|
||||
return mav;
|
||||
}
|
||||
|
||||
/**
|
||||
* 写字楼楼租金地址估值
|
||||
*
|
||||
* @param model
|
||||
* @param request
|
||||
* @return
|
||||
*/
|
||||
public Object addressEvaluateSellAction(PropertyEvaluationModel model, HttpServletRequest request) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
if (StringUtils.isBlank(model.getCity()) || StringUtils.isBlank(model.getAddress()) || model.getSize() <= 0) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("参数非法:城市,地址,面积(size)为必填项"));
|
||||
return mav;
|
||||
}
|
||||
if (model.getCity().endsWith("市")) {
|
||||
model.setCity(model.getCity().replaceAll("市$", ""));
|
||||
}
|
||||
|
||||
if (model.getCarportPrice() != null && (model.getCarportPrice() < 0 || model.getCarportPrice() > 1000000)) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("carportPrice(车位)取值不合法"));
|
||||
return mav;
|
||||
}
|
||||
if (model.getMaxFloor() != null && (model.getMaxFloor() <= 0 || model.getMaxFloor() > 59)) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("maxFloor取值非法,正确范围[1,60)"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
if (model.getFloor() != null && (model.getFloor() <= 0 || model.getFloor() > 59)) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("floor取值非法,正确范围[1,60)"));
|
||||
return mav;
|
||||
}
|
||||
if (model.getMaxFloor() != null && model.getFloor() != null && model.getFloor() > model.getMaxFloor()) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("所在楼层不能大于总楼层"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
ShopEvaluationDto evaluation = aiShopService.addressEvaluateSell(buildEvaluation(model, request));
|
||||
|
||||
if (evaluation.getUnitPrice() != null && evaluation.getUnitPrice() != 0) {
|
||||
Map<String, Object> districtPriceMap = new HashMap<String, Object>();
|
||||
districtPriceMap.put("unitPrice", evaluation.getUnitPrice());
|
||||
districtPriceMap.put("city", evaluation.getCity());
|
||||
districtPriceMap.put("district", evaluation.getDistrict());
|
||||
districtPriceMap.put("name", evaluation.getName());
|
||||
districtPriceMap.put("samples", evaluation.getSeawater());
|
||||
districtPriceMap.put("location", evaluation.getLocation());
|
||||
|
||||
mav.setModel(BasicModel.buildOKModel().setResult(districtPriceMap));
|
||||
try {
|
||||
if (StringUtils.isNotBlank(evaluation.getFromUserId()) && !evaluation.getFromUserId().equals("1007497")) {
|
||||
openApiLogService.saveLog(CommonUtils.convertToMap(model), districtPriceMap, evaluation, "/shop/addressEvaluateSell");
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
return mav;
|
||||
} else {
|
||||
mav.setModel(BasicModel.buildOKModel().setMsg("暂时不支持该地址写字楼的评估").markNoData());
|
||||
return mav;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 申请估值报告
|
||||
*
|
||||
* @param model 申请报告请求参数
|
||||
* @return
|
||||
*/
|
||||
public Object assessPriceReportApplyAction(ReportApplyModel model) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
if (StringUtils.isBlank(model.getClient())) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("参数非法:委托机构为必填项"));
|
||||
return mav;
|
||||
}
|
||||
if (StringUtils.isBlank(model.getSubjectMatter())) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("参数非法:标的物为必填项"));
|
||||
return mav;
|
||||
}
|
||||
if (StringUtils.isBlank(model.getCity()) || model.getCity().contains("全国")) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("参数非法:城市为必填项且不能为全国"));
|
||||
return mav;
|
||||
}
|
||||
if (StringUtils.isBlank(model.getDistrict()) || model.getDistrict().contains("全国")) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("参数非法:区域为必填项且不能为全国"));
|
||||
return mav;
|
||||
}
|
||||
if (model.getSize() == null || model.getSize() <= 0 ) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("参数非法:面积(size)为必填项且大于0"));
|
||||
return mav;
|
||||
}
|
||||
if (model.getCity().endsWith("市")) {
|
||||
model.setCity(model.getCity().replaceAll("市$", ""));
|
||||
}
|
||||
if (model.getDistrict().endsWith("区")) {
|
||||
model.setDistrict(model.getDistrict().replaceAll("区$", ""));
|
||||
}
|
||||
if (model.getMaxFloor() != null && (model.getMaxFloor() <= 0 || model.getMaxFloor() > 59)) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("maxFloor取值非法,正确范围[1,60)"));
|
||||
return mav;
|
||||
}
|
||||
if (model.getFloor() != null && (model.getFloor() <= 0 || model.getFloor() > 59)) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("floor取值非法,正确范围[1,60)"));
|
||||
return mav;
|
||||
}
|
||||
if (model.getMaxFloor() != null && model.getFloor() != null && model.getFloor() > model.getMaxFloor()) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("所在楼层不能大于总楼层"));
|
||||
return mav;
|
||||
}
|
||||
HttpRequest request = new HttpRequest();
|
||||
JSONObject json = null;
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("client", model.getClient());
|
||||
params.put("subjectMatter", model.getSubjectMatter());
|
||||
params.put("city", model.getCity());
|
||||
params.put("address", StringUtils.isNotBlank(model.getAddress()) ? model.getAddress() : null);
|
||||
params.put("district", model.getDistrict());
|
||||
params.put("name", StringUtils.isNotBlank(model.getName()) ? model.getName() : null);
|
||||
params.put("houseNum", StringUtils.isNotBlank(model.getBuildingNumber()) ? model.getBuildingNumber() : null);
|
||||
params.put("roomNum", StringUtils.isNotBlank(model.getUnitNumber()) ? model.getUnitNumber() : null);
|
||||
params.put("size", String.valueOf(model.getSize()));
|
||||
params.put("houseFloor", model.getFloor() != null ? String.valueOf(model.getFloor()) : null);
|
||||
params.put("houseHeight", model.getMaxFloor() != null ? String.valueOf(model.getMaxFloor()) : null);
|
||||
params.put("buildType", StringUtils.isNotBlank(model.getPropertyType()) ? model.getPropertyType() : "0");
|
||||
params.put("direction", StringUtils.isNotBlank(model.getDirection()) ? model.getDirection() : "3");
|
||||
params.put("fitment", StringUtils.isNotBlank(model.getDecoration()) ? model.getDecoration() : "2");
|
||||
JSONObject jsonObject = JSONObject.fromObject(params);
|
||||
Map<String, String> headers = new HashMap<String, String>();
|
||||
headers.put("Authorization", "Bearer " + FangJiaTongUrl.FANG_JIA_TONG_TOKEN);
|
||||
headers.put("Content-Type", "application/json");
|
||||
try {
|
||||
String ret = request.postWithBody(FangJiaTongUrl.FANG_JIA_TONG_URL_UAT + FangJiaTongUrl.ASSESS_PRICE_REPORT_APPLY, jsonObject, headers, "UTF-8");
|
||||
json = JSONObject.fromObject(FangJiaTongUrl.replaceAll(ret));
|
||||
} catch (Exception e) {
|
||||
try {
|
||||
String ret = request.postWithBody(FangJiaTongUrl.FANG_JIA_TONG_URL_UAT + FangJiaTongUrl.ASSESS_PRICE_REPORT_APPLY, jsonObject, headers, "UTF-8");
|
||||
json = JSONObject.fromObject(FangJiaTongUrl.replaceAll(ret));
|
||||
} catch (Exception ex) {
|
||||
ex.printStackTrace();
|
||||
}
|
||||
}
|
||||
if (json != null) {
|
||||
if (json.getInt("code") == 200 && json.containsKey("data")) {
|
||||
HashMap<String, String> result = new HashMap<String, String>();
|
||||
result.put("caseId", String.valueOf(json.get("data")));
|
||||
mav.setModel(BasicModel.buildOKModel().setResult(result));
|
||||
} else {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("申请估值报告失败:" + json.getString("message")));
|
||||
}
|
||||
} else {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("申请估值报告失败:调用申请报告接口返回为空"));
|
||||
}
|
||||
return mav;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取估值报告
|
||||
*
|
||||
* @param caseId 估值报告ID
|
||||
* @return
|
||||
*/
|
||||
public Object assessPriceReportFetchAction(@Param(name = "caseId") String caseId) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
if (StringUtils.isBlank(caseId)) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("参数非法:估值报告业务编号为必填项"));
|
||||
return mav;
|
||||
}
|
||||
HttpRequest request = new HttpRequest();
|
||||
JSONObject json = null;
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("caseId", caseId);
|
||||
JSONObject jsonObject = JSONObject.fromObject(params);
|
||||
Map<String, String> headers = new HashMap<String, String>();
|
||||
headers.put("Authorization", "Bearer " + FangJiaTongUrl.FANG_JIA_TONG_TOKEN);
|
||||
headers.put("Content-Type", "application/json");
|
||||
try {
|
||||
String ret = request.postWithBody(FangJiaTongUrl.FANG_JIA_TONG_URL_UAT + FangJiaTongUrl.ASSESS_PRICE_REPORT_FETCH, jsonObject, headers, "UTF-8");
|
||||
json = JSONObject.fromObject(FangJiaTongUrl.replaceAll(ret));
|
||||
} catch (Exception e) {
|
||||
try {
|
||||
String ret = request.postWithBody(FangJiaTongUrl.FANG_JIA_TONG_URL_UAT + FangJiaTongUrl.ASSESS_PRICE_REPORT_FETCH, jsonObject, headers, "UTF-8");
|
||||
json = JSONObject.fromObject(FangJiaTongUrl.replaceAll(ret));
|
||||
} catch (Exception ex) {
|
||||
ex.printStackTrace();
|
||||
}
|
||||
}
|
||||
if (json != null) {
|
||||
if (json.getInt("code") == 200 && json.containsKey("data")) {
|
||||
JSONObject resultJson = json.getJSONObject("data");
|
||||
mav.setModel(BasicModel.buildOKModel().setResult(resultJson));
|
||||
} else {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("获取估值报告失败:" + json.getString("message")));
|
||||
}
|
||||
} else {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("申请报告失败:调用获取估值报告接口返回为空"));
|
||||
}
|
||||
return mav;
|
||||
}
|
||||
|
||||
private Integer getFloor(String roomNumber) {
|
||||
if (StringUtils.isNotBlank(roomNumber) && DataUtil.isNumber(roomNumber)) {
|
||||
if (roomNumber.length() == 3) {
|
||||
return Integer.valueOf(roomNumber.substring(0, 1));
|
||||
} else if (roomNumber.length() == 4) {
|
||||
return Integer.valueOf(roomNumber.substring(0, 2));
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 通过EvaluationModel组装Evaluation实体
|
||||
*
|
||||
* @param model
|
||||
* @param request
|
||||
* @return
|
||||
*/
|
||||
private Evaluation buildEvaluation(PropertyEvaluationModel model, HttpServletRequest request) {
|
||||
Evaluation evaluation = new Evaluation();
|
||||
evaluation.setLevel(model.getLevel());
|
||||
evaluation.setFitment(model.getFitment());
|
||||
evaluation.setType(model.getType());
|
||||
|
||||
evaluation.setCity(model.getCity());
|
||||
evaluation.setRegion(model.getRegion());
|
||||
evaluation.setDistrictName(model.getName());
|
||||
evaluation.setAddress(model.getAddress());
|
||||
evaluation.setDirection(model.getDirection());
|
||||
evaluation.setNoise(NOISE.get(model.getNoise()));
|
||||
evaluation.setType(model.getType());
|
||||
evaluation.setSight(SIGHT.get(model.getView()));
|
||||
evaluation.setAtticArea(model.getAtticSize());
|
||||
evaluation.setTerraceArea(model.getOuterSize());
|
||||
evaluation.setBasementArea(model.getBasementSize());
|
||||
evaluation.setGardenArea(model.getGardenSize());
|
||||
String worryFacility = model.getNegativeFactor();
|
||||
evaluation.setWorryFacility(NEGATIVE_FACTOR.get(worryFacility));
|
||||
evaluation.setCreationDate(new Date());
|
||||
evaluation.setModificationDate(new Date());
|
||||
evaluation.setClientNo(model.getClientNo());
|
||||
if (0 != model.getAge()) {
|
||||
evaluation.setAge(model.getAge());
|
||||
}
|
||||
evaluation.setArea(model.getSize());
|
||||
if (ArrayUtils.isNotEmpty(model.getBuildQuality())) {
|
||||
evaluation.setBuildQuality(convertToString(model.getBuildQuality()));
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getBuildType())) {
|
||||
evaluation.setBuildType(model.getBuildType());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getCarport())) {
|
||||
evaluation.setCarport(model.getCarport());
|
||||
}
|
||||
if (model.getDecorationAge() >= 1 && model.getDecorationAge() <= 5) {
|
||||
evaluation.setFitmentAge(model.getDecorationAge());
|
||||
}
|
||||
evaluation.setCarportPrice(model.getCarportPrice());
|
||||
evaluation.setCity(model.getCity());
|
||||
evaluation.setDistrictName(model.getName());
|
||||
if (model.getDecorationCost() != null && model.getDecorationCost() > 0) {
|
||||
evaluation.setFitmentCharge(model.getDecorationCost());
|
||||
}
|
||||
if (0 != model.getHall()) {
|
||||
evaluation.setHall(model.getHall());
|
||||
}
|
||||
if (null != model.getFloor()) {
|
||||
evaluation.setFloor(model.getFloor());
|
||||
}
|
||||
if (model.getMaxFloor() != null && 0 != model.getMaxFloor()) {
|
||||
evaluation.setHeight(model.getMaxFloor());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getBuildingNumber())) {
|
||||
evaluation.setHouseNum(model.getBuildingNumber());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getHouseProperty())) {
|
||||
evaluation.setHouseProperty(model.getHouseProperty());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getInsidepos())) {
|
||||
evaluation.setInsidepos(model.getInsidepos());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getLightingVentilation())) {
|
||||
evaluation.setLightingVentilation(model.getLightingVentilation());
|
||||
}
|
||||
|
||||
if (StringUtils.isNotBlank(model.getOtherFactor())) {
|
||||
evaluation.setOtherFactor(model.getOtherFactor());
|
||||
}
|
||||
evaluation.setRegion(model.getDistrict());
|
||||
if (null != model.getRoom()) {
|
||||
evaluation.setRoom(model.getRoom());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getRoomNumber())) {
|
||||
evaluation.setRoomNum(model.getRoomNumber());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getSideset())) {
|
||||
evaluation.setSideset(model.getSideset());
|
||||
}
|
||||
if (0 != model.getToilet()) {
|
||||
evaluation.setToilet(model.getToilet());
|
||||
}
|
||||
if (model.getFacilities() != null) {
|
||||
String facilities = model.getFacilities();
|
||||
String[] fs = facilities.split(",");
|
||||
for (int i = 0; i < fs.length; i++) {
|
||||
fs[i] = FACILITIES.get(fs[i]);
|
||||
}
|
||||
evaluation.setFacilities(convertToString(fs));
|
||||
}
|
||||
|
||||
if (StringUtils.isNotBlank(model.getFacade())) {
|
||||
evaluation.setFacade(model.getFacade());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getTraffice())) {
|
||||
evaluation.setTraffice(model.getTraffice());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getShape())) {
|
||||
evaluation.setShape(model.getShape());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getObstacle())) {
|
||||
evaluation.setObstacle(model.getObstacle());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getIndustry())) {
|
||||
evaluation.setIndustry(model.getIndustry());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getSupportDevice())) {
|
||||
evaluation.setSupportDevice(model.getSupportDevice());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getLeftPosition())) {
|
||||
evaluation.setLeftPosition(model.getLeftPosition());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getShopType())) {
|
||||
evaluation.setShopType(model.getShopType());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getShopUnique())) {
|
||||
evaluation.setShopUnique(model.getShopUnique());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getLocation())) {
|
||||
evaluation.setLocation(model.getLocation());
|
||||
}
|
||||
if (model.isNewDistrict()) {
|
||||
evaluation.setNewDistrict(model.isNewDistrict());
|
||||
}
|
||||
String floorDesc = "中";
|
||||
if (evaluation.getFloor() != null && evaluation.getHeight() != null) {
|
||||
Double floorInfo = evaluation.getFloor() * 1.0 / evaluation.getHeight();
|
||||
if (floorInfo <= 0.33) {
|
||||
floorDesc = "低";
|
||||
} else if (floorInfo > 0.66) {
|
||||
floorDesc = "高";
|
||||
}
|
||||
evaluation.setFloorDesc(floorDesc);
|
||||
}
|
||||
String ip = AnalyticsUtil.getIpAddress(ActionContext.getContext().getRequest());
|
||||
evaluation.setIp(ip);
|
||||
AuthenticationBean authenticationBean = (AuthenticationBean) request
|
||||
.getAttribute(AuthenticationBean.ATTR_AUTHENTICATION_BEAN);
|
||||
if (authenticationBean != null && authenticationBean.apiUser != null) {
|
||||
evaluation.setFromUserId(String.valueOf(authenticationBean.apiUser.getId()));
|
||||
}
|
||||
evaluation.setSource("api_shop_evaluateRent");
|
||||
return evaluation;
|
||||
}
|
||||
|
||||
private static final Map<String, String> FACILITIES = new HashMap<String, String>() {
|
||||
{
|
||||
put("0", "健身房");
|
||||
put("1", "游泳池");
|
||||
put("2", "会所");
|
||||
put("3", "网球场");
|
||||
put("4", "球场");
|
||||
put("5", "棋牌室");
|
||||
put("6", "停车场");
|
||||
put("7", "图书馆");
|
||||
put("8", "活动中心");
|
||||
}
|
||||
};
|
||||
|
||||
public void setOpenApiLogService(OpenApiLogService openApiLogService) {
|
||||
this.openApiLogService = openApiLogService;
|
||||
}
|
||||
|
||||
public void setFieldFilterService(FieldFilterService fieldFilterService) {
|
||||
this.fieldFilterService = fieldFilterService;
|
||||
}
|
||||
|
||||
public void setAiShopService(AiShopService aiShopService) {
|
||||
this.aiShopService = aiShopService;
|
||||
}
|
||||
}
|
||||
@@ -543,6 +543,18 @@
|
||||
<version>1.18.2</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-text</artifactId>
|
||||
<version>1.9</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.hankcs.hanlp.restful</groupId>
|
||||
<artifactId>hanlp-restful</artifactId>
|
||||
<version>0.0.12</version>
|
||||
</dependency>
|
||||
|
||||
<!-- <dependency>-->
|
||||
<!-- <groupId>com.fangjia</groupId>-->
|
||||
<!-- <artifactId>fangjia-open</artifactId>-->
|
||||
|
||||
+1
-1
@@ -21,7 +21,7 @@ import java.sql.DriverManager
|
||||
@BatchOwner(Watcher.WangWenYan)
|
||||
class ClickHouseDistrictStatMarkupCalcBatch extends BaseBatch {
|
||||
|
||||
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 27017)
|
||||
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo", 27017)
|
||||
static void main(String[] args) {
|
||||
runBatch(new ClickHouseDistrictStatMarkupCalcBatch(), args)
|
||||
}
|
||||
|
||||
+1
-1
@@ -18,7 +18,7 @@ import com.mongodb.DBCollection
|
||||
@BatchOwner(Watcher.WangWenYan)
|
||||
class DistrictStatMarkupCalcBatch extends BaseBatch {
|
||||
|
||||
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 27017)
|
||||
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com", 27017)
|
||||
Date oneYearAgo = DateUtil.add(new Date(), Calendar.MONTH, -1)
|
||||
|
||||
|
||||
|
||||
+3
-1
@@ -8,8 +8,10 @@ import com.mongodb.DBCollection
|
||||
/**
|
||||
* 深圳公众号成交数据录入
|
||||
*/
|
||||
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2025年01月深圳二手房成交的.xlsx", 0, 1)
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
//没有成交时间,随机生成
|
||||
|
||||
+1
-1
@@ -18,7 +18,7 @@ import java.util.concurrent.atomic.AtomicLong
|
||||
*/
|
||||
class AddThirdPriceSample {
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
|
||||
public static final DBCollection NO_PRICE_DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "deal_price_all", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
+1
-1
@@ -43,7 +43,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
+1
-1
@@ -45,7 +45,7 @@ class DistrictDelete {
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictDelete()
|
||||
|
||||
+1
-1
@@ -27,7 +27,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
|
||||
// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07",
|
||||
|
||||
+1
-1
@@ -36,7 +36,7 @@ class TradeStatLog {
|
||||
def pssqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
|
||||
|
||||
+4
-4
@@ -18,10 +18,10 @@ import com.mongodb.QueryOperators
|
||||
import groovy.sql.Sql
|
||||
|
||||
|
||||
DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
|
||||
DBCollection CITY_STAT = MongoUtils.getCollectionByName("fangjia_stat", "city_stat", "192.168.0.43")
|
||||
DBCollection CITY_INFO = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", "192.168.0.43")
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
|
||||
DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
DBCollection CITY_STAT = MongoUtils.getCollectionByName("fangjia_stat", "city_stat", "mongo.db.fangjia.com")
|
||||
DBCollection CITY_INFO = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", "mongo.db.fangjia.com")
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
+1
-1
@@ -12,7 +12,7 @@ class MatchDealPriceRegion {
|
||||
|
||||
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018)
|
||||
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
static main(args) {
|
||||
|
||||
+1
-1
@@ -21,7 +21,7 @@ import org.apache.lucene.queryparser.classic.QueryParser
|
||||
*/
|
||||
class EvaluationBatch {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
+1
-1
@@ -18,7 +18,7 @@ class FumingTest {
|
||||
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302
|
||||
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
|
||||
+1
-1
@@ -38,7 +38,7 @@ class HailanTest {
|
||||
}
|
||||
|
||||
static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
|
||||
static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ import java.util.concurrent.atomic.AtomicLong
|
||||
class PocStat {
|
||||
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", lineUrl)
|
||||
public static final DBCollection LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia_base", "loudong", lineUrl)
|
||||
|
||||
|
||||
+1
-1
@@ -15,7 +15,7 @@ class RuiFenPocTest {
|
||||
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302
|
||||
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
|
||||
+1
-1
@@ -99,7 +99,7 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
public static final DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009")
|
||||
public static final DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
public static final DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com", 28017)
|
||||
public static final DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com")
|
||||
|
||||
public static final DBCollection STATS_INDEX_COLL = MongoUtils.getCollectionByName("fangjia_stat", "stats_index", "mongo.db.fangjia.com")
|
||||
|
||||
+1
-1
@@ -23,7 +23,7 @@ import org.apache.commons.lang.StringUtils
|
||||
class FewerHouseDistrictCrawlCommandBatch extends BaseBatch {
|
||||
private static
|
||||
final DBCollection ARTEMIS_COLL = MongoUtils.getCollectionByName("fangjia_log", "artemis_message", "mongo.db.fangjia.com")
|
||||
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43")
|
||||
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
static void main(String[] args) {
|
||||
runBatch(new FewerHouseDistrictCrawlCommandBatch(), args)
|
||||
|
||||
@@ -15,7 +15,7 @@ import org.bson.types.ObjectId
|
||||
//new FewerHouseDistrictCrawlCommandBatch().process(null);
|
||||
//return;
|
||||
//
|
||||
//DBCollection col_43_artemis_message = MongoUtils.getCollectionByName("fangjia_log", "artemis_message", "192.168.0.43", 27017);
|
||||
//DBCollection col_43_artemis_message = MongoUtils.getCollectionByName("fangjia_log", "artemis_message", "mongo.db.fangjia.com", 27017);
|
||||
//DBCursor cursor_artemis_message = col_43_artemis_message.find(new BasicDBObject("status", 0));
|
||||
//int count = 0;
|
||||
//while(cursor_artemis_message.hasNext()){
|
||||
|
||||
+2
-2
@@ -21,8 +21,8 @@ import java.util.concurrent.TimeUnit
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class DistrictTradeHouseStatBatch extends BaseBatch{
|
||||
|
||||
public static DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
|
||||
public static DBCollection DISTRICT_TRADE_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_trade_stat", "192.168.0.43")
|
||||
public static DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
public static DBCollection DISTRICT_TRADE_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_trade_stat", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new DistrictTradeHouseStatBatch(), args)
|
||||
|
||||
+1
-1
@@ -23,7 +23,7 @@ import org.elasticsearch.common.xcontent.XContentType
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class SynchDealPriceToEs extends BaseBatch{
|
||||
static DBCollection dealPriceConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017)
|
||||
static DBCollection dealPriceConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com",28017)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new SynchDealPriceToEs(), args)
|
||||
|
||||
+1
-1
@@ -20,7 +20,7 @@ class AbcEvaluationRowsBatch extends BaseBatch{
|
||||
|
||||
static final def fjtSqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/goojia_fangjia?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver");
|
||||
|
||||
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", dbUrl)
|
||||
|
||||
static main(args) {
|
||||
|
||||
+1
-1
@@ -29,7 +29,7 @@ class DistrictInfoExportBatch extends BaseBatch{
|
||||
|
||||
def charset = "GBK"
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection FZ_ADDR_COLL = MongoUtils.getCollectionByName2("business", "fz_district_address", lineUrl)
|
||||
public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName2("business", "fz_district", lineUrl)
|
||||
|
||||
|
||||
+1
-1
@@ -23,7 +23,7 @@ class FzDataResultStat {
|
||||
|
||||
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName2("business", "fz_district", lineUrl)
|
||||
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ import net.sf.json.JSONObject
|
||||
import org.apache.commons.io.FileUtils
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
|
||||
|
||||
@@ -169,7 +169,7 @@ DingDingMessageUtil.sendTextMessage(access_token, s)
|
||||
//
|
||||
//
|
||||
//def count(name, sDate){
|
||||
// DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
// DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("city","¸£ÖÝ")
|
||||
// q.put("s_date", sDate)
|
||||
|
||||
+1
-1
@@ -21,7 +21,7 @@ class ThirdPriceExportBatch extends BaseBatch{
|
||||
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
|
||||
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ import java.util.concurrent.CountDownLatch
|
||||
|
||||
class ThirdTest {
|
||||
|
||||
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
|
||||
static main(args) {
|
||||
|
||||
+1
-1
@@ -25,7 +25,7 @@ class TradeExportBatch extends BaseBatch{
|
||||
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
|
||||
DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
|
||||
static def districtMap = [:]
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@ import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
class CzBankStat {
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("business", "average_statistics", "192.168.0.43")
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("business", "average_statistics", "mongo.db.fangjia.com")
|
||||
def static final DBCollection CZLOG = MongoUtils.getCollectionByName("fangjia_log", "cz_log2", "192.168.0.136")
|
||||
static main(args) {
|
||||
def task = new CzBankStat();
|
||||
|
||||
+1
-1
@@ -16,7 +16,7 @@ import org.apache.commons.lang.StringUtils
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class DealPriceProcess {
|
||||
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
|
||||
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
def total = 0
|
||||
|
||||
static main(args) {
|
||||
|
||||
+1
-1
@@ -17,7 +17,7 @@ import java.util.concurrent.Executors
|
||||
|
||||
class DistrictDetailBatch {
|
||||
|
||||
public static DBCollection DISTRICT_DETAIL_COLL = MongoUtils.getCollectionByName("fangjia_youfang", "district_detail", "192.168.0.43")
|
||||
public static DBCollection DISTRICT_DETAIL_COLL = MongoUtils.getCollectionByName("fangjia_youfang", "district_detail", "mongo.db.fangjia.com")
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
|
||||
@@ -40,7 +40,7 @@ class EsHouseStat {
|
||||
dataSource.setProperties(connectionProperties)
|
||||
}
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
public static void main(String[] args){
|
||||
|
||||
@@ -16,8 +16,8 @@ import java.sql.ResultSet
|
||||
|
||||
class FZDataExport {
|
||||
|
||||
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
|
||||
+1
-1
@@ -8,7 +8,7 @@ import com.mongodb.DBCollection
|
||||
|
||||
class FZHouseEvaluation {
|
||||
|
||||
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", dbUrl)
|
||||
|
||||
def regionMap = [:]
|
||||
|
||||
@@ -8,7 +8,7 @@ import com.mongodb.DBCollection
|
||||
|
||||
class FjwUserStat {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
|
||||
public static final DBCollection USER_COLL = MongoUtils.getCollectionByName2("fangjia_coresite", "user", lineUrl)
|
||||
|
||||
|
||||
+1
-1
@@ -18,7 +18,7 @@ import org.bson.Document;
|
||||
import org.bson.conversions.Bson;
|
||||
|
||||
class HouseLouDongMatch {
|
||||
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", dbUrl)
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
+1
-1
@@ -22,7 +22,7 @@ class LandDetailCrawl {
|
||||
|
||||
DBCollection LAND_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "land_deal_price1", "192.168.0.235", 27018)
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
|
||||
public static final DBCollection FJT_LAND_COLL = MongoUtils.getCollectionByName2("business", "land_distinct", lineUrl)
|
||||
public static final DBCollection OFF_FJT_LAND_COLL = MongoUtils.getCollectionByName2("business", "land_distinct", offlineUrl)
|
||||
|
||||
@@ -14,10 +14,10 @@ import org.apache.lucene.queryparser.classic.QueryParser
|
||||
|
||||
class LogStat {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new LogStat()
|
||||
|
||||
+2
-2
@@ -21,8 +21,8 @@ import java.util.concurrent.Executors
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class NoHouseDistrictStatBatch extends BaseBatch{
|
||||
|
||||
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
// public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
// public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", dbUrl)
|
||||
public static final DBCollection AVERAGER_CONFIG_COLL = MongoUtils.getCollectionByName2("fangjia", "averager_config", dbUrl)
|
||||
|
||||
|
||||
+1
-1
@@ -13,7 +13,7 @@ class PingAnAddressLevalTest {
|
||||
|
||||
String token = "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB";
|
||||
|
||||
DBCollection DIST_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_grade", "192.168.0.43")
|
||||
DBCollection DIST_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_grade", "mongo.db.fangjia.com")
|
||||
static main(args) {
|
||||
def task = new PingAnAddressLevalTest()
|
||||
task.run();
|
||||
|
||||
@@ -329,7 +329,7 @@ def findDistrictPrice(def city, def region){
|
||||
q.put("region", region);
|
||||
}
|
||||
//districtStatConn.addOption(Bytes.QUERYOPTION_SLAVEOK);
|
||||
final def districtStatConn = Mongo3Utils.getDbCollection("mongodb://developer:G00IDCDEV12385@192.168.0.43:28017","fangjia_stat", "district_stat");
|
||||
final def districtStatConn = Mongo3Utils.getDbCollection("mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017","fangjia_stat", "district_stat");
|
||||
def count = districtStatConn.count(q)
|
||||
return count;
|
||||
}
|
||||
@@ -343,7 +343,7 @@ def findDistrictPrice(def city, def region){
|
||||
q.put("region", region);
|
||||
}
|
||||
//districtStatConn.addOption(Bytes.QUERYOPTION_SLAVEOK);
|
||||
final def districtStatConn = Mongo3Utils.getDbCollection("mongodb://developer:G00IDCDEV12385@192.168.0.43:28017","fangjia_stat", "district_stat");
|
||||
final def districtStatConn = Mongo3Utils.getDbCollection("mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017","fangjia_stat", "district_stat");
|
||||
def count = districtStatConn.count(q)
|
||||
return count;
|
||||
}
|
||||
+1
-1
@@ -6,7 +6,7 @@ import com.mongodb.DBCollection
|
||||
|
||||
class RegionBlockSetting {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection REGION_BLOCK_COLL = MongoUtils.getCollectionByName2("fangjia", "region_block", lineUrl)
|
||||
|
||||
static main(args) {
|
||||
|
||||
+2
-2
@@ -17,8 +17,8 @@ import com.mongodb.DBObject
|
||||
class SynchDistrictStatBatch extends BaseBatch{
|
||||
|
||||
|
||||
//public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 28017)
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
//public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com", 28017)
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
|
||||
|
||||
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", lineUrl)
|
||||
|
||||
+1
-1
@@ -15,7 +15,7 @@ import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class UpdateThirdPrice {
|
||||
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
def static final DBCollection AVERAGE_STATISTICS_OFFLINE = CrawalDBUtil.getAverage_statistics_col();
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
|
||||
|
||||
+1
-1
@@ -14,7 +14,7 @@ import groovy.sql.Sql
|
||||
class ApiUserCountAnalyBatch extends BaseBatch{
|
||||
|
||||
static final dbFangjia = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia?autoReconnect=true&characterEncoding=utf-8&autoReconnect=true", "sa", "G00jia", "com.mysql.jdbc.Driver");
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
DBCollection AUTHORITY_COLL = MongoUtils.getCollectionByName2("fangjia", "authority.api_user_authority", lineUrl)
|
||||
DBCollection VISIT_COLL = MongoUtils.getCollectionByName2("business", "api_visit_count", lineUrl)
|
||||
|
||||
|
||||
+1
-1
@@ -10,7 +10,7 @@ import com.mongodb.DBCollection
|
||||
|
||||
class CcbCityPriceStat {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
static main(args) {
|
||||
|
||||
+1
-1
@@ -20,7 +20,7 @@ import com.mongodb.DBCollection
|
||||
*/
|
||||
class PocTest2023 {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
|
||||
|
||||
public static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
|
||||
|
||||
+1
-1
@@ -15,7 +15,7 @@ import org.apache.commons.lang.math.RandomUtils
|
||||
class DistrictSelectHouse {
|
||||
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictSelectHouse();
|
||||
|
||||
+2
-2
@@ -9,8 +9,8 @@ import com.mongodb.QueryOperators
|
||||
|
||||
class TestPrice {
|
||||
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
static main(args) {
|
||||
|
||||
+1
-1
@@ -17,7 +17,7 @@ import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
class CityStatExport {
|
||||
|
||||
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
|
||||
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", dbUrl)
|
||||
|
||||
|
||||
+1
-1
@@ -16,7 +16,7 @@ import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class CityStat {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", lineUrl)
|
||||
|
||||
public static final DBCollection BAD_CASE_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "bad_case", lineUrl)
|
||||
|
||||
+3
-3
@@ -21,13 +21,13 @@ import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
|
||||
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
|
||||
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
|
||||
DBCollection region_stat_coll = MongoUtils.getCollectionByName("fangjia", "region_stat", "192.168.0.43")
|
||||
DBCollection region_stat_coll = MongoUtils.getCollectionByName("fangjia", "region_stat", "mongo.db.fangjia.com")
|
||||
DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020)
|
||||
|
||||
//def citys = ["ÎÚ³ľÆë","²ý¼ª","°¢¿ËËÕ","ÒÁÀç","¿¦Ê²"]
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@ import com.mongodb.QueryOperators
|
||||
class BestDistrictPriceExport {
|
||||
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new BestDistrictPriceExport();
|
||||
|
||||
+378
-68
@@ -2,12 +2,18 @@ package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CopyOnWriteArraySet
|
||||
@@ -16,30 +22,47 @@ import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class AutoAddOfficeBatch extends BaseBatch{
|
||||
class AutoAddOfficeBatch extends BaseBatch {
|
||||
|
||||
def fullCityMap = CommonDatautils.queryFullNameByCity()
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
def fullCityMap = CommonDatautils.queryFullNameByCity()
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206", 27020)
|
||||
|
||||
//235为房价网内部计算均价的库
|
||||
def pgsqlInstance235 = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
// runBatch(new AutoAddOfficeBatch(), args)
|
||||
|
||||
def task = new AutoAddOfficeBatch()
|
||||
task.repeatOffice()
|
||||
// task.repeatOffice()
|
||||
task.repeatDist()
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
|
||||
// def pattern = /[A-Z0-9]座|[0-9a-z]栋|[A-Z]栋|·|\(建设中\)|[0-9]号楼|\(装修中\)|[A-Z]楼|[0-9]幢/
|
||||
// def newName = "开放广场".replaceAll(pattern, "")
|
||||
// println newName
|
||||
// findAddOfficeDatas() //查找需要新增的写字楼数据
|
||||
// checkInBaiduMapOffice()//查寻在百度地图上有的写字楼数据
|
||||
// addOffice()//新增写字楼
|
||||
|
||||
|
||||
}
|
||||
|
||||
def findAddOfficeDatas() {
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def allAddOffices = []
|
||||
allAddOffices << ["地图ID","省","市","区","写字楼名称","地址","经度","纬度","匹配城市","匹配区域","匹配名称","匹配方式"]
|
||||
for(def c : citys){
|
||||
allAddOffices << ["地图ID", "省", "市", "区", "写字楼名称", "地址", "经度", "纬度", "匹配城市", "匹配区域", "匹配名称", "匹配方式"]
|
||||
for (def c : citys) {
|
||||
def cityObj = fullCityMap.get(c)
|
||||
if(cityObj){
|
||||
if (cityObj) {
|
||||
def officeMaps = CommonDatautils.findOfficeByMap(cityObj.city, "商务写字楼")
|
||||
officeMaps = filterOffice(officeMaps)
|
||||
if(officeMaps){
|
||||
if (officeMaps) {
|
||||
def offices = matchOffice(officeMaps)
|
||||
|
||||
allAddOffices.addAll(offices)
|
||||
@@ -49,59 +72,72 @@ class AutoAddOfficeBatch extends BaseBatch{
|
||||
ExcelUtils.write("D:\\数据\\高德写字楼.xlsx", allAddOffices, "数据")
|
||||
}
|
||||
|
||||
def filterOffice(officeMaps){
|
||||
def checkInBaiduMapOffice() {
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\高德写字楼1.xlsx", { d ->
|
||||
if (!d['L']) {
|
||||
def obj = findBaiduDatas(d['C'], d['D'], d['E'])
|
||||
if (obj) {
|
||||
d['L'] = "百度存在"
|
||||
d['M'] = obj.std_tag.toString()
|
||||
}
|
||||
}
|
||||
|
||||
}, 0)
|
||||
}
|
||||
|
||||
def filterOffice(officeMaps) {
|
||||
def offices = []
|
||||
def keys = []
|
||||
for(def o : officeMaps){
|
||||
if(!o){
|
||||
for (def o : officeMaps) {
|
||||
if (!o) {
|
||||
continue
|
||||
}
|
||||
if(o.name.startsWith("写字楼") || o.name.startsWith("综合楼") || o.name.contains("销售") || o.name.startsWith("办公楼") || o.name.startsWith("公务楼") || o.name.startsWith("行政楼") || o.name.startsWith("商务办公楼") || o.name.startsWith("综合办公楼")){
|
||||
println "[ 过滤 ] "+o.city+" "+o.name
|
||||
if (o.name.startsWith("写字楼") || o.name.startsWith("综合楼") || o.name.contains("销售") || o.name.startsWith("办公楼") || o.name.startsWith("公务楼") || o.name.startsWith("行政楼") || o.name.startsWith("商务办公楼") || o.name.startsWith("综合办公楼")) {
|
||||
println "[ 过滤 ] " + o.city + " " + o.name
|
||||
continue
|
||||
}
|
||||
if(o.name.contains("路写字楼")){
|
||||
println "[ 过滤 ] "+o.city+" "+o.name
|
||||
if (o.name.contains("路写字楼")) {
|
||||
println "[ 过滤 ] " + o.city + " " + o.name
|
||||
continue
|
||||
}
|
||||
|
||||
def pattern = /[A-Z0-9]座|[0-9a-z]栋|[A-Z]栋|·|\(建设中\)|[0-9]号楼|\(装修中\)|[A-Z]楼|[0-9]幢/
|
||||
def newName = o.name.replaceAll(pattern, "")
|
||||
o.name = newName
|
||||
def keyName = o.city +"#"+o.district+"#"+o.name
|
||||
if(!keys.contains(keyName)){
|
||||
def keyName = o.city + "#" + o.district + "#" + o.name
|
||||
if (!keys.contains(keyName)) {
|
||||
offices << o
|
||||
keys << keyName
|
||||
}
|
||||
}
|
||||
println "原:"+officeMaps.size() +" 新:"+offices.size()
|
||||
println "原:" + officeMaps.size() + " 新:" + offices.size()
|
||||
return offices
|
||||
}
|
||||
|
||||
def matchOffice(offices){
|
||||
def matchOffice(offices) {
|
||||
|
||||
def addCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(offices.size())
|
||||
def total = offices.size()
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
|
||||
for(def i=0;i<offices.size();i++){
|
||||
for (def i = 0; i < offices.size(); i++) {
|
||||
final def nmd = offices[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
try {
|
||||
addCount.addAndGet(1)
|
||||
|
||||
def office = matchDB(nmd)
|
||||
if(!office){
|
||||
if (!office) {
|
||||
office = matchApi(nmd)
|
||||
}
|
||||
if(office){
|
||||
if (office) {
|
||||
rets << [nmd.map_id, nmd.province, nmd.city, nmd.district, nmd.name, nmd.address, nmd.lng, nmd.lat, office.city, office.region, office.name, office.match_way]
|
||||
println addCount.get()+"/"+total +" [success] "+ office.get("region")+" "+office.get("name")+" "+office.match_way
|
||||
}else{
|
||||
rets << [nmd.map_id, nmd.province, nmd.city, nmd.district, nmd.name, nmd.address,nmd.lng, nmd.lat,"","","",""]
|
||||
println addCount.get()+"/"+total +" [fail] "+nmd.city+" "+nmd.name
|
||||
println addCount.get() + "/" + total + " [success] " + office.get("region") + " " + office.get("name") + " " + office.match_way
|
||||
} else {
|
||||
rets << [nmd.map_id, nmd.province, nmd.city, nmd.district, nmd.name, nmd.address, nmd.lng, nmd.lat, "", "", "", ""]
|
||||
println addCount.get() + "/" + total + " [fail] " + nmd.city + " " + nmd.name
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
@@ -115,28 +151,28 @@ class AutoAddOfficeBatch extends BaseBatch{
|
||||
return rets
|
||||
}
|
||||
|
||||
def matchDB(nmd){
|
||||
def matchDB(nmd) {
|
||||
def simpleCity = RegionData.formatCity(nmd.city)
|
||||
def simpleDistrict = RegionData.formatRegion(simpleCity, nmd.district)
|
||||
if(simpleCity && simpleDistrict){
|
||||
if (simpleCity && simpleDistrict) {
|
||||
def office = null
|
||||
def rets = CommonDatautils.findDistrictList(simpleCity, simpleDistrict, nmd.name, "office")
|
||||
if(rets){
|
||||
if(rets.size()>=1){
|
||||
if (rets) {
|
||||
if (rets.size() >= 1) {
|
||||
office = rets[0]
|
||||
}
|
||||
}else{
|
||||
} else {
|
||||
rets = CommonDatautils.findDistrictByAlias(simpleCity, simpleDistrict, nmd.name, "office")
|
||||
if(rets && rets.size()>=1){
|
||||
if (rets && rets.size() >= 1) {
|
||||
office = rets[0]
|
||||
}else{
|
||||
} else {
|
||||
rets = CommonDatautils.findDistrictByAddress(simpleCity, simpleDistrict, nmd.name, "district")
|
||||
if(rets && rets.size()>=1){
|
||||
if (rets && rets.size() >= 1) {
|
||||
office = rets[0]
|
||||
}
|
||||
}
|
||||
}
|
||||
if(office){
|
||||
if (office) {
|
||||
office.match_way = "数据库"
|
||||
}
|
||||
return office
|
||||
@@ -144,65 +180,152 @@ class AutoAddOfficeBatch extends BaseBatch{
|
||||
return null
|
||||
}
|
||||
|
||||
def matchApi(nmd){
|
||||
def matchApi(nmd) {
|
||||
def address = nmd.city
|
||||
if(nmd.district){
|
||||
if (nmd.district) {
|
||||
address = address + nmd.district
|
||||
}
|
||||
address = address + nmd.name
|
||||
def jsonObject = FangjiaClientUtils.newMatch(nmd.city, address, "office", false)
|
||||
if(!jsonObject){
|
||||
if(DataUtil.isPreciseAddress(nmd.address)){
|
||||
if (!jsonObject) {
|
||||
if (DataUtil.isPreciseAddress(nmd.address)) {
|
||||
address = nmd.city
|
||||
if(nmd.district){
|
||||
if (nmd.district) {
|
||||
address = address + nmd.district
|
||||
}
|
||||
address = address + nmd.address
|
||||
jsonObject = FangjiaClientUtils.newMatch(nmd.city, address, "office", false)
|
||||
}
|
||||
}
|
||||
if(jsonObject) {
|
||||
def udata = [:]
|
||||
udata.put("_id",jsonObject.get("id"))
|
||||
udata.put("city",jsonObject.get("city"))
|
||||
udata.put("region",jsonObject.get("district"))
|
||||
udata.put("name",jsonObject.get("name"))
|
||||
udata.put("match_type",jsonObject.get("propertyType"))
|
||||
udata.put("match_credit",jsonObject.get("credit"))
|
||||
if (jsonObject) {
|
||||
def udata = [:]
|
||||
udata.put("_id", jsonObject.get("id"))
|
||||
udata.put("city", jsonObject.get("city"))
|
||||
udata.put("region", jsonObject.get("district"))
|
||||
udata.put("name", jsonObject.get("name"))
|
||||
udata.put("match_type", jsonObject.get("propertyType"))
|
||||
udata.put("match_credit", jsonObject.get("credit"))
|
||||
udata.put("match_way", "接口")
|
||||
return udata
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def repeatOffice(){
|
||||
def repeatOffice() {
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def allrets = []
|
||||
allrets << ["ID","城市","区域","名称","地址","可疑ID","可疑城市","可疑区域","可疑名称","可疑地址","相似度","距离"]
|
||||
for(def c : citys){
|
||||
def offices = CommonDatautils.findAllDistricts(c, "office", ["city","region","name","address", "lat", "lng"])
|
||||
allrets << ["ID", "城市", "区域", "名称", "地址", "可疑ID", "可疑城市", "可疑区域", "可疑名称", "可疑地址", "相似度", "距离"]
|
||||
for (def c : citys) {
|
||||
def offices = CommonDatautils.findAllDistricts(c, "office", ["city", "region", "name", "address", "lat", "lng"])
|
||||
def rets = checkRepeatOfficeBatch(offices)
|
||||
println c +" "+offices.size()
|
||||
println c + " " + offices.size()
|
||||
allrets.addAll(rets)
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\写字楼\\重复可疑写字楼1.xlsx", allrets, "数据")
|
||||
}
|
||||
|
||||
def checkRepeatOfficeBatch(offices){
|
||||
def repeatDist(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def allrets = []
|
||||
allrets << [ "城市", "区域", "名称", "别名","计算方式","地址","名称相似度","地址相似度","总相似度"]
|
||||
def index = 0
|
||||
for (def c : citys) {
|
||||
index++
|
||||
if(index>4){
|
||||
continue
|
||||
}
|
||||
def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "address", "alias"])
|
||||
def rets = checkRepeatDistrictBatch(c, districts)
|
||||
println c + " " + districts.size()
|
||||
allrets.addAll(rets)
|
||||
println "${index}"+ c +" 运A行完成 "+rets.size()
|
||||
}
|
||||
ExcelUtils.write("/home/goojia/gj/重复可疑小区1.xlsx", allrets, "数据")
|
||||
}
|
||||
|
||||
def checkRepeatDistrictBatch(city,districts) {
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def cityId = cityCodeDataMap.get(city)
|
||||
|
||||
def fjwPriceMap = findNewFjwAvgPriceList(cityId, 202502)
|
||||
def addCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
def total = districts.size()
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def sets = new CopyOnWriteArraySet()
|
||||
for (def i = 0; i < total; i++) {
|
||||
final def o1 = districts[i]
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
try {
|
||||
addCount.addAndGet(1)
|
||||
// if(!o1.name.contains("四公司")){
|
||||
// continue
|
||||
// }
|
||||
if(!sets.contains(o1._id.toString())){
|
||||
def similarRets = findListSimilar(o1, districts)
|
||||
if (similarRets) {
|
||||
def newPriceMap1 = fjwPriceMap.get(o1._id.toString())
|
||||
def new_price1 = null
|
||||
if(newPriceMap1){
|
||||
new_price1 = newPriceMap1.calc_type
|
||||
}
|
||||
def ailas_str1 = ""
|
||||
if(o1.alias){
|
||||
ailas_str1 = o1.alias.join(",")
|
||||
}
|
||||
def rows = []
|
||||
for(def similar : similarRets){
|
||||
if (!sets.contains(similar._id.toString())) {
|
||||
sets.add(similar._id.toString())
|
||||
def ailas_str = ""
|
||||
if(similar.alias){
|
||||
ailas_str = similar.alias.join(",")
|
||||
}
|
||||
def newPriceMap = fjwPriceMap.get(o1._id.toString())
|
||||
def new_price = null
|
||||
if(newPriceMap){
|
||||
new_price = newPriceMap.calc_type
|
||||
}
|
||||
rows << [ similar.city, similar.region, similar.name, ailas_str,new_price, similar.address, similar.nameScore, similar.addrScore, similar.score]
|
||||
// rets << [ similar.city, similar.region, similar.name, ailas_str,new_price, similar.address, similar.nameScore, similar.addrScore, similar.score]
|
||||
}
|
||||
}
|
||||
if(rows){
|
||||
sets.add(o1._id.toString())
|
||||
rets << [ o1.city, o1.region, o1.name, ailas_str1,new_price1, o1.address, "主", "主", "主"]
|
||||
rets.addAll(rows)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
// }
|
||||
// });
|
||||
}
|
||||
// latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
def checkRepeatOfficeBatch(offices) {
|
||||
def addCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(offices.size())
|
||||
def total = offices.size()
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def sets = new CopyOnWriteArraySet()
|
||||
for(def i=0;i<total;i++){
|
||||
for (def i = 0; i < total; i++) {
|
||||
final def o1 = offices[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
try {
|
||||
addCount.addAndGet(1)
|
||||
def maxObj = findOneSimilarOffice(o1, offices)
|
||||
if(maxObj){
|
||||
if(!sets.contains(o1._id.toString())){
|
||||
if (maxObj) {
|
||||
if (!sets.contains(o1._id.toString())) {
|
||||
sets.add(maxObj._id.toString())
|
||||
rets << [o1._id.toString(), o1.city, o1.region, o1.name, o1.address, maxObj._id.toString(), maxObj.city, maxObj.region, maxObj.name, maxObj.address, maxObj.score, maxObj.distance]
|
||||
}
|
||||
@@ -219,28 +342,215 @@ class AutoAddOfficeBatch extends BaseBatch{
|
||||
return rets
|
||||
}
|
||||
|
||||
def findOneSimilarOffice(o1, offices){
|
||||
def findListSimilar(o1, datas){
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def ids = []
|
||||
ids << o1._id.toString()
|
||||
for (def o2 : datas) {
|
||||
if (o1._id.toString() != o2._id.toString() && o1.region==o2.region) {
|
||||
|
||||
// if(!o2.name.contains("四公司")){
|
||||
// continue
|
||||
// }
|
||||
|
||||
def name1 = o1.name//.replace("小区","").replace("别墅","")
|
||||
def name2 = o2.name//.replace("小区","").replace("别墅","")
|
||||
def nameScore = CosineSimilarAlgorithm.levenshteinSimilarity(name1, name2);
|
||||
def addrScore = null
|
||||
if(o1.address && o2.address){
|
||||
addrScore = CosineSimilarAlgorithm.levenshteinSimilarity(o1.address, o2.address);
|
||||
}
|
||||
def str = null
|
||||
if(o1.name.length() > o2.name.length()){
|
||||
str = o1.name.replace(o2.name, "")
|
||||
}else{
|
||||
str = o2.name.replace(o1.name, "")
|
||||
}
|
||||
if (nameScore > 0.6 && !["一期","二期","三期","四期","五期","六期","七期","八期","九期","十期","东区","西区","南区","北区","A区","B区","C区","D区"].contains(str)) {
|
||||
o2.nameScore = nameScore
|
||||
o2.addrScore = addrScore
|
||||
o2.score = nameScore
|
||||
rets << o2
|
||||
ids << o2._id.toString()
|
||||
def maxObj = findMaxSimilar(ids, o2, datas)
|
||||
while(maxObj){
|
||||
maxObj.nameScore = CosineSimilarAlgorithm.levenshteinSimilarity(maxObj.name, o1.name);
|
||||
if(o1.address && maxObj.address){
|
||||
maxObj.addrScore = CosineSimilarAlgorithm.levenshteinSimilarity(o1.address, maxObj.address);
|
||||
}
|
||||
maxObj.score = maxObj.nameScore
|
||||
rets << maxObj
|
||||
ids << maxObj._id.toString()
|
||||
maxObj = findMaxSimilar(ids, maxObj, datas)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
rets.sort{a,b->
|
||||
def a1 = a.score
|
||||
def b1 = b.score
|
||||
return b1.compareTo(a1)
|
||||
}
|
||||
|
||||
if (rets.size()>2) {
|
||||
return rets[0..2]
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
def findMaxSimilar(ids, o1, datas){
|
||||
def maxObj = null
|
||||
def maxScore = -1000
|
||||
for (def o2 : datas) {
|
||||
if (o1._id.toString() != o2._id.toString() && o1.region==o2.region && !ids.contains(o2._id.toString())) {
|
||||
def name1 = o1.name//.replace("小区","").replace("别墅","")
|
||||
def name2 = o2.name//.replace("小区","").replace("别墅","")
|
||||
def nameScore = CosineSimilarAlgorithm.levenshteinSimilarity(name1, name2);
|
||||
def addrScore = null
|
||||
if(o1.address && o2.address){
|
||||
addrScore = CosineSimilarAlgorithm.levenshteinSimilarity(o1.address, o2.address);
|
||||
}
|
||||
def str = null
|
||||
if(o1.name.length() > o2.name.length()){
|
||||
str = o1.name.replace(o2.name, "")
|
||||
}else{
|
||||
str = o2.name.replace(o1.name, "")
|
||||
}
|
||||
if (nameScore > 0.6 && nameScore>maxScore && !["一期","二期","三期","四期","五期","六期","七期","八期","九期","十期","东区","西区","南区","北区","A区","B区","C区","D区"].contains(str)) {
|
||||
maxObj = o2
|
||||
o2.nameScore = nameScore
|
||||
o2.addrScore = addrScore
|
||||
o2.score = nameScore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return maxObj
|
||||
}
|
||||
|
||||
def findOneSimilarOffice(o1, offices) {
|
||||
def maxScore = 0
|
||||
def maxObj = null
|
||||
for(def o2 : offices){
|
||||
if(o1._id.toString()!=o2._id.toString()){
|
||||
for (def o2 : offices) {
|
||||
if (o1._id.toString() != o2._id.toString()) {
|
||||
double nameScore = CosineSimilarAlgorithm.getSimilarity(o1.name, o2.name);
|
||||
double addrScore = CosineSimilarAlgorithm.getSimilarity(o1.address,o2.address);
|
||||
if(nameScore+addrScore>maxScore){
|
||||
maxScore = nameScore+addrScore
|
||||
double addrScore = CosineSimilarAlgorithm.getSimilarity(o1.address, o2.address);
|
||||
if (nameScore + addrScore > maxScore) {
|
||||
maxScore = nameScore + addrScore
|
||||
maxObj = o2
|
||||
}
|
||||
}
|
||||
}
|
||||
def distance = -1000
|
||||
if(o1.lat && maxObj && maxObj.lat){
|
||||
distance = GeoHelper.GetDistance(o1.lat/10000000000.0, o1.lng/10000000000.0, maxObj.lat/10000000000.0, maxObj.lng/10000000000.0)
|
||||
if (o1.lat && maxObj && maxObj.lat) {
|
||||
distance = GeoHelper.GetDistance(o1.lat / 10000000000.0, o1.lng / 10000000000.0, maxObj.lat / 10000000000.0, maxObj.lng / 10000000000.0)
|
||||
}
|
||||
if(maxObj){
|
||||
if (maxObj) {
|
||||
maxObj.distance = distance
|
||||
maxObj.score = maxScore
|
||||
return maxObj
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
def findBaiduDatas(city, district, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("district", district)
|
||||
q.put("name", name)
|
||||
// q.put('std_tag', '房地产;住宅区')
|
||||
def obj = BAIDU_POI_COLL.findOne(q)
|
||||
return obj;
|
||||
}
|
||||
|
||||
def addOffice(){
|
||||
ExcelUtils.modify("D:\\数据\\高德待入库写字楼.xlsx",{d->
|
||||
if(d['L']!='人工删除'){
|
||||
def city = RegionData.formatCity(d['C'])
|
||||
def region = RegionData.formatRegion(city, d['D'])
|
||||
if(city && region){
|
||||
def locationstr = d['G'].toString()+","+d['H'].toString()
|
||||
def bdpoint = FangjiaClientUtils.convert(locationstr, 1)
|
||||
def udata = new BasicDBObject()
|
||||
if(bdpoint){
|
||||
udata.put("lng2", (bdpoint[0]*10000000000).longValue())
|
||||
udata.put("lat2", (bdpoint[1]*10000000000).longValue())
|
||||
}
|
||||
|
||||
udata.put("city", city)
|
||||
udata.put("region", region)
|
||||
udata.put("name", d['E'])
|
||||
udata.put("address", d['I'])
|
||||
udata.put("cat", "office")
|
||||
udata.put("status", 0)
|
||||
udata.put("c_date",new Date())
|
||||
udata.put("m_date",new Date())
|
||||
udata.put("lng", (d['G'].toString().toDouble()*10000000000).longValue())
|
||||
udata.put("lat", (d['H'].toString().toDouble()*10000000000).longValue())
|
||||
|
||||
def estate_type = null
|
||||
if(d['M'] && d['M'].contains("园区")){
|
||||
estate_type = "创意园区"
|
||||
}
|
||||
|
||||
if(!estate_type && d['E'].contains("创意")){
|
||||
estate_type = "创意园区"
|
||||
}
|
||||
if(estate_type){
|
||||
udata.put("estate_type2", estate_type)
|
||||
}
|
||||
|
||||
def markMap = [:]
|
||||
markMap.put("data_source", "机器入库")
|
||||
udata.put("mark_map", markMap)
|
||||
DbUtils.seaweed.save(udata)
|
||||
println d['E'] +" "+udata._id.toString()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def findNewFjwAvgPrice(def commid, def cityid){
|
||||
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from price_stat.avm_community_base_price where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
|
||||
def map = [:]
|
||||
pgsqlInstance235.eachRow(sql, { r ->
|
||||
map.avg_price = r.gujia_bds_clc_avprc
|
||||
map.type = r.gujia_bds_clmtd
|
||||
map.month = r.gujiabds_clc_yrmo_val
|
||||
map.credit_value = r.gujia_bds_cfdc_dgr
|
||||
if(r.gujia_bds_clmtd == 1){
|
||||
map.calc_type = "人工估值"
|
||||
}else if(r.gujia_bds_clmtd == 2){
|
||||
map.calc_type = "样本计算"
|
||||
}else if(r.gujia_bds_clmtd == 3){
|
||||
map.calc_type = "神经网络"
|
||||
}
|
||||
})
|
||||
return map
|
||||
}
|
||||
|
||||
def findNewFjwAvgPriceList(def cityid, date){
|
||||
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr,gujia_bds_id from price_stat.avm_community_base_price where urbn_id='${cityid}' and gujiabds_clc_yrmo_val=${date}"
|
||||
def retMap = [:]
|
||||
pgsqlInstance235.eachRow(sql, { r ->
|
||||
def map = [:]
|
||||
map.avg_price = r.gujia_bds_clc_avprc
|
||||
map.type = r.gujia_bds_clmtd
|
||||
map.month = r.gujiabds_clc_yrmo_val
|
||||
map.credit_value = r.gujia_bds_cfdc_dgr
|
||||
if(r.gujia_bds_clmtd == 1){
|
||||
map.calc_type = "人工估值"
|
||||
}else if(r.gujia_bds_clmtd == 2){
|
||||
map.calc_type = "样本计算"
|
||||
}else if(r.gujia_bds_clmtd == 3){
|
||||
map.calc_type = "神经网络"
|
||||
}
|
||||
retMap.put(r.gujia_bds_id, map)
|
||||
})
|
||||
return retMap
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+1
-2
@@ -5,10 +5,8 @@ import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* Ãû¼¼¯
|
||||
*/
|
||||
@@ -34,6 +32,7 @@ class MinglujiData {
|
||||
def prov = "½ËÕÊ¡"
|
||||
def city = "ÎÞÎý"
|
||||
exportDatas(city)
|
||||
// exportDatas(city)
|
||||
// matchDev(prov, city)
|
||||
// matchCommunity(city)
|
||||
// fiveLevelSplit(city)
|
||||
|
||||
+57
-6
@@ -4,9 +4,12 @@ import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.apache.commons.io.FileUtils
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.update.UpdateRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
@@ -33,10 +36,14 @@ class RegionStandardBatch extends BaseBatch{
|
||||
void process(Object args) {
|
||||
seaweedRegionStandard()
|
||||
// poiRegionStandard()
|
||||
// seaweedRegionStandard()
|
||||
// poiRegionStandard()
|
||||
//通过乡镇街道修改区域 如:东莞市
|
||||
// updateRegionByTown()
|
||||
}
|
||||
|
||||
def seaweedRegionStandard(){
|
||||
seaweedRegionStandardByCat("district")
|
||||
seaweedRegionStandardByCat("office")
|
||||
}
|
||||
def poiRegionStandard(){
|
||||
|
||||
@@ -93,7 +100,7 @@ class RegionStandardBatch extends BaseBatch{
|
||||
|
||||
|
||||
def seaweedRegionStandardByCat(cat){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def citys = ["伊春"]//CommonDatautils.findAllCity()
|
||||
for(def city : citys){
|
||||
|
||||
def rMap = regionMap.get(city)
|
||||
@@ -148,8 +155,12 @@ class RegionStandardBatch extends BaseBatch{
|
||||
def streetMap = CommonDatautils.queryAllStreetNameByCity(fullCityName)
|
||||
if(streetMap){
|
||||
set = streetMap.keySet()
|
||||
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name"])
|
||||
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name","lng","lat"])
|
||||
println city +" "+ districts.size()
|
||||
def index = 0
|
||||
for(def d : districts){
|
||||
index ++
|
||||
println index +" "+ d.region
|
||||
if(set.contains(d.region)){
|
||||
continue
|
||||
}
|
||||
@@ -157,8 +168,12 @@ class RegionStandardBatch extends BaseBatch{
|
||||
def updateRegion = null
|
||||
def flag = null
|
||||
for(def key : set){
|
||||
def alias = streetMap.get(key)
|
||||
if((alias && alias.contains(d.region)) || (!alias && key.contains(d.region))){
|
||||
def obj = streetMap.get(key)
|
||||
def alias = null
|
||||
if(obj && obj.alias){
|
||||
alias = obj.alias
|
||||
}
|
||||
if((alias && alias.contains(d.region)) || (!alias && key==d.region)){
|
||||
def newRegion = RegionData.formatRegion(d.city, key)
|
||||
if(newRegion!=d.region){
|
||||
updateRegion = newRegion
|
||||
@@ -166,13 +181,26 @@ class RegionStandardBatch extends BaseBatch{
|
||||
flag = "pass"
|
||||
break;
|
||||
}
|
||||
|
||||
if(obj.shape){
|
||||
def inAreaFlag = InsidePolygon.isInArea(d.lng/10000000000.0, d.lat/10000000000.0, InsidePolygon.createPoint4(obj.shape.coordinates))
|
||||
if(inAreaFlag){
|
||||
def newRegion = obj.street
|
||||
newRegion = RegionData.formatRegion(d.city, newRegion)
|
||||
if(newRegion!=d.region){
|
||||
updateRegion = newRegion
|
||||
}
|
||||
flag = "pass"
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if(updateRegion){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("region", updateRegion)
|
||||
udata.put("m_date", new Date())
|
||||
println d.city +" "+d.region+" "+d.name +" ÇøÓòÐÞ¸ÄΪ£º"+updateRegion
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
if(flag!="pass"){
|
||||
def udata = new BasicDBObject()
|
||||
@@ -402,4 +430,27 @@ class RegionStandardBatch extends BaseBatch{
|
||||
return rets;
|
||||
}
|
||||
|
||||
def updateRegionByTown(){
|
||||
ExcelUtils.modify("d:\\数据\\城市与区域相同写字楼.xlsx",{d->
|
||||
def town = RegionData.formatRegion(d['B'], d['F'])
|
||||
if(town){
|
||||
|
||||
def obj = CommonDatautils.findDistrict(d['B'], town, d['E'], "office")
|
||||
if(!obj){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("region", town)
|
||||
udata.put("m_date", new Date())
|
||||
println d['F']+" "+" 区域修改为:" + town
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("status", 1)
|
||||
println d['F']+" "+" 删除:" + town
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata))
|
||||
}
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -16,7 +16,7 @@ class RegionUpdateBatch {
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new RegionUpdateBatch()
|
||||
|
||||
+17
-13
@@ -1,11 +1,13 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
@@ -23,12 +25,15 @@ class SynchAdministrativeMappingDataBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def indexName = "administrative_mapping"
|
||||
def retMap = synchAdminiMappingTo100()
|
||||
synchToEs(retMap, "192.168.0.89")
|
||||
// synchToEs(retMap, "192.168.0.52")
|
||||
synchToEs(retMap, "192.168.0.89", indexName)
|
||||
synchToEs(retMap, "192.168.0.52", indexName)
|
||||
println(" ÔËÐÐÍê³É")
|
||||
|
||||
}
|
||||
|
||||
def synchAdminiMappingTo100(){
|
||||
def synchAdminiMappingTo100(city){
|
||||
//删除旧的,再同步新的
|
||||
ADMINISTRATIVE_MAPPING_COLL_100.remove(new BasicDBObject())
|
||||
|
||||
@@ -52,12 +57,11 @@ class SynchAdministrativeMappingDataBatch extends BaseBatch{
|
||||
return retMap;
|
||||
}
|
||||
|
||||
def synchToEs(retMap, host){
|
||||
def synchToEs(retMap, host, indexName){
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery()
|
||||
|
||||
def oldIds = new CopyOnWriteArrayList()
|
||||
|
||||
ElasticUtils89.executeQuery(host, "administrative_mapping", queryBuilder, { String id, doc->
|
||||
ElasticUtils89.queryUpdate(host, indexName, queryBuilder, { String id, doc->
|
||||
oldIds << id
|
||||
def mongoObj = retMap.get(id)
|
||||
def updateFieldCount = 0
|
||||
@@ -65,8 +69,7 @@ class SynchAdministrativeMappingDataBatch extends BaseBatch{
|
||||
if(!["id","_id","c_date","m_date","polyline_source","operator","city_simple_pinyin","prefecture_level_city"].contains(k)){
|
||||
def val = doc.get(k)
|
||||
if(val!=v){
|
||||
// updateValMap.put(k,val)
|
||||
if(v.type!="MultiPolygon"){
|
||||
if(k!="shape" || (k=="shape" && v.type!="MultiPolygon")){
|
||||
println k +" 修改 "+id
|
||||
updateFieldCount++
|
||||
doc.put(k,v)
|
||||
@@ -80,14 +83,14 @@ class SynchAdministrativeMappingDataBatch extends BaseBatch{
|
||||
}else{
|
||||
if(!mongoObj){
|
||||
println "删除:"+id +" "+doc.name
|
||||
ElasticUtils.delete(id, host, "administrative_mapping")
|
||||
ElasticUtils.delete(id, host, indexName)
|
||||
return false
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
def addEsDatas = []
|
||||
def addEsDatasMap = [:]
|
||||
retMap.each{id,map->
|
||||
if(!oldIds.contains(id)){
|
||||
println "需新增:"+id
|
||||
@@ -95,11 +98,12 @@ class SynchAdministrativeMappingDataBatch extends BaseBatch{
|
||||
map.remove("m_date")
|
||||
map.remove("polyline_source")
|
||||
map.remove("operator")
|
||||
addEsDatas << map
|
||||
map.remove("_id")
|
||||
addEsDatasMap.put(id, map)
|
||||
}
|
||||
}
|
||||
if(addEsDatas){
|
||||
ElasticUtils89.updateEs(addEsDatas, host, "administrative_mapping")
|
||||
if(addEsDatasMap){
|
||||
ElasticUtils89.updateEs(addEsDatasMap, host, indexName)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+30
-6
@@ -10,6 +10,7 @@ import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
|
||||
/**
|
||||
* 将43上的开发区同步到mongo-100, es-89, es-52
|
||||
@@ -27,9 +28,10 @@ class SynchDevDataBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def indexName = "development_zone"
|
||||
def devMap = synchDevTo100()
|
||||
synchDevEs(devMap, "192.168.0.89")
|
||||
synchDevEs(devMap, "192.168.0.52")
|
||||
synchDevEs(devMap, "192.168.0.89", indexName)
|
||||
synchDevEs(devMap, "192.168.0.52", indexName)
|
||||
}
|
||||
|
||||
def synchDevTo100(){
|
||||
@@ -81,10 +83,12 @@ class SynchDevDataBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
|
||||
def synchDevEs(devMap, host){
|
||||
def synchDevEs(devMap, host, indexName){
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery()
|
||||
def oldIds = new CopyOnWriteArrayList()
|
||||
|
||||
ElasticUtils89.executeQuery(host, "development_zone", queryBuilder, { String id, doc->
|
||||
ElasticUtils89.executeQuery(host, indexName, queryBuilder, { String id, doc->
|
||||
oldIds << id
|
||||
def mongoObj = devMap.get(id)
|
||||
def updateFieldCount = 0
|
||||
mongoObj.each{k,v->
|
||||
@@ -92,7 +96,6 @@ class SynchDevDataBatch extends BaseBatch{
|
||||
def val = doc.get(k)
|
||||
if(val!=v){
|
||||
println doc.name +" 修改 "+k+" "+ v+" "+id
|
||||
// updateValMap.put(k,val)
|
||||
updateFieldCount++
|
||||
doc.put(k,v)
|
||||
}
|
||||
@@ -104,11 +107,32 @@ class SynchDevDataBatch extends BaseBatch{
|
||||
}else{
|
||||
if(!mongoObj){
|
||||
println "删除:"+id +" "+doc.name
|
||||
ElasticUtils.delete(id, host, "development_zone")
|
||||
ElasticUtils.delete(id, host, indexName)
|
||||
return false
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
def addEsDatasMap = [:]
|
||||
devMap.each{id,map->
|
||||
if(!oldIds.contains(id)){
|
||||
println "ÐèÐÂÔö£º"+id
|
||||
map.remove("create_time")
|
||||
map.remove("build_date")
|
||||
map.remove("bd_shape")
|
||||
map.remove("img_url")
|
||||
map.remove("operator")
|
||||
map.remove("operate_result")
|
||||
map.remove("m_date")
|
||||
map.remove("township")
|
||||
map.remove("block")
|
||||
map.remove("_id")
|
||||
addEsDatasMap.put(id, map)
|
||||
}
|
||||
}
|
||||
if(addEsDatasMap){
|
||||
ElasticUtils89.updateEs(addEsDatasMap, host, indexName)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
+16
-8
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
@@ -219,34 +220,41 @@ class TempExportData {
|
||||
}
|
||||
|
||||
def exportOfficeHouseCount(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\写字楼\\待核实写字楼.xlsx",{d->
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\高德写字楼-房源第三方验证.xlsx",{d->
|
||||
// def obj = findYuanQu(d['B'], d['D'])
|
||||
// if(obj){
|
||||
// d['T'] = "ǰհ´æÔÚ"
|
||||
// }
|
||||
// if(!d['T']){
|
||||
// def obj = findYuanQu(d['B'], d['D'])
|
||||
// def city = RegionData.formatCity(d['C'])
|
||||
// def obj = findYuanQu(city, d['E'])
|
||||
// if(obj){
|
||||
// d['T'] = "前瞻存在"
|
||||
// d['U'] = obj.name
|
||||
// }
|
||||
// }
|
||||
// if(!d['T']) {
|
||||
// def count = findHouseCountByCat(d['B'], d['C'], d['D'], "office_sell")
|
||||
// def city = RegionData.formatCity(d['C'])
|
||||
// def region = RegionData.formatRegion(city, d['D'])
|
||||
//
|
||||
// def count = findHouseCountByCat(city, region, d['E'], "office_sell")
|
||||
// if (!count) {
|
||||
// count = findHouseCountByCat(d['B'], d['C'], d['D'], "office_rent")
|
||||
// count = findHouseCountByCat(city, region, d['E'], "office_rent")
|
||||
// }
|
||||
// d['S'] = count
|
||||
// d['T'] = count
|
||||
// }
|
||||
|
||||
if(!d['J']){
|
||||
def datas = CommonDatautils.findThirdNetWorkOffices(d['B'], d['C'], d['D'])
|
||||
if(!d['T']){
|
||||
def city = RegionData.formatCity(d['C'])
|
||||
def region = RegionData.formatRegion(city, d['D'])
|
||||
def datas = CommonDatautils.findThirdNetWorkOffices(city, region, d['E'])
|
||||
if(datas){
|
||||
def sources = []
|
||||
for(def o : datas){
|
||||
sources << o.source
|
||||
}
|
||||
if(sources){
|
||||
d['K'] = sources.join(",")
|
||||
d['V'] = sources.join(",")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+52
-53
@@ -1,58 +1,57 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.apache.commons.lang.math.RandomUtils
|
||||
|
||||
def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName2("fangjia", "region_stat", mongoDbUrl)
|
||||
|
||||
|
||||
def city = "昌江"
|
||||
ExcelUtils.modify("D:\\数据\\海南\\海南省城市均价.xlsx",{d->
|
||||
def date = d['B']
|
||||
def start = DateUtils.getFirstDayOfMonth(date)
|
||||
def end = DateUtils.getLastDayOfMonth(date)
|
||||
def weekends = DateUtils.getWeekendBetweenDates(start, end)
|
||||
for(def weekend : weekends){
|
||||
def avg_price = d['I']
|
||||
def val = random(-200,200)
|
||||
def new_price = (int)(avg_price*val/10000 + avg_price)
|
||||
def last_weekend = DateUtils.add(DateUtils.formatDateByPattern(weekend, "yyyy-MM-dd"), -7)
|
||||
def last_price = CommonDatautils.queryCityPriceByWeekend(city, last_weekend)
|
||||
|
||||
def o = new BasicDBObject()
|
||||
o.put("avg_price", new_price)
|
||||
o.put("c_date" , new Date())
|
||||
o.put("city", city)
|
||||
o.put("m_date" , new Date())
|
||||
o.put("weekend", weekend)
|
||||
o.put("cat","城市")
|
||||
o.put("moving_average" , (int)avg_price)
|
||||
if(!last_price){
|
||||
o.put("weekend_markup" , 0)
|
||||
}else{
|
||||
def weekend_markup = (int)((new_price-last_price)*100/last_price)
|
||||
o.put("weekend_markup" , weekend_markup)
|
||||
}
|
||||
|
||||
def q = new BasicDBObject("cat", "城市").append("city", city).append("weekend", weekend)
|
||||
REGION_STAT_COLL.update(q, new BasicDBObject('$set', o),true, false)
|
||||
}
|
||||
},1)
|
||||
|
||||
|
||||
public static final int random(int i1, int i2) {
|
||||
int max = Math.max(i1, i2);
|
||||
int min = Math.min(i1, i2);
|
||||
return min + RandomUtils.nextInt(max - min + 1);
|
||||
}
|
||||
|
||||
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
|
||||
ObjectId objectId = new ObjectId("659c9ae92ae7732fd7d54e40");
|
||||
Date creationDate = objectId.getDate();
|
||||
System.out.println(creationDate);
|
||||
|
||||
//def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
//DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName2("fangjia", "region_stat", mongoDbUrl)
|
||||
//
|
||||
//
|
||||
//def city = "昌江"
|
||||
//ExcelUtils.modify("D:\\数据\\海南\\海南省城市均价.xlsx",{d->
|
||||
// def date = d['B']
|
||||
// def start = DateUtils.getFirstDayOfMonth(date)
|
||||
// def end = DateUtils.getLastDayOfMonth(date)
|
||||
// def weekends = DateUtils.getWeekendBetweenDates(start, end)
|
||||
// for(def weekend : weekends){
|
||||
// def avg_price = d['I']
|
||||
// def val = random(-200,200)
|
||||
// def new_price = (int)(avg_price*val/10000 + avg_price)
|
||||
// def last_weekend = DateUtils.add(DateUtils.formatDateByPattern(weekend, "yyyy-MM-dd"), -7)
|
||||
// def last_price = CommonDatautils.queryCityPriceByWeekend(city, last_weekend)
|
||||
//
|
||||
// def o = new BasicDBObject()
|
||||
// o.put("avg_price", new_price)
|
||||
// o.put("c_date" , new Date())
|
||||
// o.put("city", city)
|
||||
// o.put("m_date" , new Date())
|
||||
// o.put("weekend", weekend)
|
||||
// o.put("cat","城市")
|
||||
// o.put("moving_average" , (int)avg_price)
|
||||
// if(!last_price){
|
||||
// o.put("weekend_markup" , 0)
|
||||
// }else{
|
||||
// def weekend_markup = (int)((new_price-last_price)*100/last_price)
|
||||
// o.put("weekend_markup" , weekend_markup)
|
||||
// }
|
||||
//
|
||||
// def q = new BasicDBObject("cat", "城市").append("city", city).append("weekend", weekend)
|
||||
// REGION_STAT_COLL.update(q, new BasicDBObject('$set', o),true, false)
|
||||
// }
|
||||
//},1)
|
||||
//
|
||||
//
|
||||
//public static final int random(int i1, int i2) {
|
||||
// int max = Math.max(i1, i2);
|
||||
// int min = Math.min(i1, i2);
|
||||
// return min + RandomUtils.nextInt(max - min + 1);
|
||||
//}
|
||||
//
|
||||
//
|
||||
//
|
||||
//
|
||||
//
|
||||
|
||||
+98
-5
@@ -3,23 +3,22 @@ package com.fangjia.script.gj.data_handle
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.GeocodeUtils
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.BaiduMapUtils
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.ProvinceData
|
||||
import com.mongodb.AggregationOutput
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.client.AggregateIterable
|
||||
import org.bson.Document
|
||||
import net.sf.json.JSONObject
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 乡镇街道中间表
|
||||
* 包含市,区,乡镇街道别名,来源 统计局community表
|
||||
@@ -45,6 +44,8 @@ class TownMiddleProcess {
|
||||
|
||||
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
|
||||
|
||||
static DBCollection POLYLINECOLL2025 = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline_2025")
|
||||
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
|
||||
static main(args) {
|
||||
@@ -96,6 +97,12 @@ class TownMiddleProcess {
|
||||
// checkCenterInTownRang()//检测乡镇街道中心点是否在围栏中
|
||||
|
||||
// addSpecialCityTownAlias()
|
||||
|
||||
|
||||
// updateNewRegionPoyline()//更新新的区域围栏
|
||||
|
||||
updateMappingPolyline()
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -681,7 +688,7 @@ class TownMiddleProcess {
|
||||
}
|
||||
|
||||
def regionPolyline(){
|
||||
def dbcursor = REGIONPOLYLINE_COLL.find(new BasicDBObject("level","district"))
|
||||
def dbcursor = POLYLINECOLL2025.find(new BasicDBObject("level","district"))
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
def resultMap = [:]
|
||||
@@ -1356,4 +1363,90 @@ class TownMiddleProcess {
|
||||
keys.put("street", 1)
|
||||
return ADMINISTRATIVE_MAPPING_COLL.find(q, keys).toList()
|
||||
}
|
||||
|
||||
def updateNewRegionPoyline(){
|
||||
|
||||
def regionPolylineMap = regionPolyline()
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","区域")
|
||||
// q.put("city", "苏州市")
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
for(def d : dbcursor){
|
||||
def shapeData = regionPolylineMap.get(d.city+"|@|"+d.district)
|
||||
println d.city+" "+d.district
|
||||
if(shapeData){
|
||||
println "新围栏数:"+shapeData.shape.coordinates.size()
|
||||
}
|
||||
|
||||
if(d.shape){
|
||||
println "旧围栏数:"+d.shape.coordinates.size()
|
||||
}
|
||||
|
||||
if(d.shape && shapeData && shapeData.shape!=d.shape){
|
||||
def udata = new BasicDBObject()
|
||||
udata.shape = shapeData.shape
|
||||
udata.location = shapeData.location
|
||||
udata.m_date = new Date()
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
|
||||
// if(shapeData && d.shape && shapeData.shape.type=="polygon" && d.shape.type=="polygon"){
|
||||
// double area = com.fangjia.script.util.GeoHelper.calculateArea(shapeData.shape.coordinates[0])
|
||||
// double area1 = com.fangjia.script.util.GeoHelper.calculateArea(d.shape.coordinates[0])
|
||||
// if(Math.abs(area1-area)>=1){
|
||||
// println "新:"+area
|
||||
// println "旧:"+area1
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.shape = shapeData.shape
|
||||
// udata.location = shapeData.location
|
||||
// //DMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
|
||||
// }
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def updateMappingPolyline(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","区域")
|
||||
q.put("shape.type","polygon")
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
|
||||
for(def d : dbcursor){
|
||||
updateAdministrativeMapping(d._id.toString(), d.shape)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def updateAdministrativeMapping(def id, def shape){
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.89:9200/administrative_mapping/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
|
||||
def body = """
|
||||
{
|
||||
"script": {
|
||||
"source": "ctx._source.shape = params.newshape",
|
||||
"params":{
|
||||
"newshape":{
|
||||
"type":"${shape.type}",
|
||||
"coordinates":${shape.coordinates}
|
||||
}
|
||||
}
|
||||
},
|
||||
"query": {
|
||||
"term": {
|
||||
"_id": "${id}"
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
try{
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
def response = JSONObject.fromObject(res)
|
||||
println(response)
|
||||
}catch(Exception e){
|
||||
println "error : "+e.getMessage()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
+1
-2
@@ -31,7 +31,6 @@ import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 区域标准化
|
||||
* 1.小区库乡镇街道标准化,包含小区,商铺,写字楼
|
||||
@@ -629,7 +628,7 @@ class TownStandardBatch extends BaseBatch{
|
||||
|
||||
// //通过调取地图api接口获取乡镇街道
|
||||
// if(!saveObj.township){
|
||||
// def gaodeMap = BaiduMapUtils.geocoding(location.lat+","+location.lon)
|
||||
def gaodeMap = BaiduMapUtils.geocoding(location.lat+","+location.lon)
|
||||
// if(gaodeMap && gaodeMap.addressComponent) {
|
||||
// def addressComponent = gaodeMap.addressComponent
|
||||
// if(addressComponent){
|
||||
|
||||
-7
@@ -1,16 +1,9 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.addrepo.InvokeHttpRequest
|
||||
import com.fangjia.script.statistics.poi.AmapPoi
|
||||
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
|
||||
import com.fangjia.script.temp.shenjian.util.HttpUtils
|
||||
import com.fangjia.script.util.HttpRequestUtil
|
||||
import com.mongodb.DBObject
|
||||
import net.sf.json.JSONArray
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.collections.CollectionUtils
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
class FangjiaClientUtils {
|
||||
|
||||
|
||||
+1
-1
@@ -9,7 +9,7 @@ import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
def city = "³£ÖÝ"
|
||||
|
||||
+1
-1
@@ -14,7 +14,7 @@ import org.apache.lucene.queryparser.classic.QueryParser
|
||||
|
||||
class EsMonthHouseStat {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
static main(args) {
|
||||
|
||||
+1
-1
@@ -38,7 +38,7 @@ class FjwBigdataDealBatch extends BaseBatch{
|
||||
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
|
||||
|
||||
+1
-1
@@ -29,7 +29,7 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
|
||||
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
|
||||
// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07",
|
||||
|
||||
+1
-1
@@ -27,7 +27,7 @@ class GuangfaBlankSelectDistrict {
|
||||
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
|
||||
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new GuangfaBlankSelectDistrict();
|
||||
|
||||
+2
-2
@@ -31,7 +31,7 @@ import java.util.concurrent.atomic.AtomicLong
|
||||
public class BeiKeCrawlByChrome {
|
||||
|
||||
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||
WebDriver driver = null;
|
||||
@@ -118,7 +118,7 @@ public class BeiKeCrawlByChrome {
|
||||
}
|
||||
|
||||
def crawlData(driver, citys){
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def runedCitys = FileUtils.readTxt("d:\\数据\\成交抓取\\贝壳已抓城市.txt")
|
||||
|
||||
|
||||
+1
-1
@@ -15,7 +15,7 @@ import java.util.regex.Pattern
|
||||
class ExportPaimai {
|
||||
|
||||
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
|
||||
|
||||
static main(args) {
|
||||
|
||||
+2
-2
@@ -31,7 +31,7 @@ import java.util.concurrent.atomic.AtomicLong
|
||||
public class LianjiaCrawlByChrome {
|
||||
|
||||
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||
WebDriver driver = null;
|
||||
@@ -189,7 +189,7 @@ public class LianjiaCrawlByChrome {
|
||||
}
|
||||
|
||||
def crawlData(driver, cityList){
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def runedCitys = FileUtils.readTxt("d:\\数据\\成交抓取\\链家已抓城市.txt")
|
||||
|
||||
|
||||
+1
-1
@@ -28,7 +28,7 @@ class PaimaiDataSend {
|
||||
DBCollection jd_coll = MongoUtils.getCollectionByName("fangjia_crawl", "fapai_jd_detail", "192.168.0.235", 27018)
|
||||
DBCollection rmfysszc_coll = MongoUtils.getCollectionByName("fangjia_crawl", "fapai_rmfysszc_detail", "192.168.0.235", 27018)
|
||||
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
+1
-1
@@ -26,7 +26,7 @@ class PaimaiTest {
|
||||
|
||||
DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai")
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
|
||||
|
||||
|
||||
|
||||
+3
-1
@@ -30,7 +30,7 @@ class AddPriceTest {
|
||||
def pgsqlInstance235 = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
public static final DBCollection confirm_price_date = MongoUtils.getCollectionByName("fangjia", "confirm_price_data", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
@@ -230,6 +230,8 @@ class AddPriceTest {
|
||||
if(newPriceMap){
|
||||
d['G'] = newPriceMap.avg_price
|
||||
d['H'] = newPriceMap.month
|
||||
d['H'] = map.calc_type
|
||||
// d['AM'] = map.credit_value
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
+2
-2
@@ -13,8 +13,8 @@ import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
class ExportToDealPrice {
|
||||
|
||||
DBCollection DEAL_PRICE_MATCH_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price_match", "192.168.0.43")
|
||||
DBCollection DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43")
|
||||
DBCollection DEAL_PRICE_MATCH_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price_match", "mongo.db.fangjia.com")
|
||||
DBCollection DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
DBCollection TRADE_MATCH_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "trade_match_price", "192.168.0.136")
|
||||
|
||||
|
||||
+1
-1
@@ -14,7 +14,7 @@ import java.util.regex.Pattern
|
||||
|
||||
class FzBlankTest {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection FZ_ADDR_COLL = MongoUtils.getCollectionByName2("business", "fz_district_address", lineUrl)
|
||||
|
||||
static main(args) {
|
||||
|
||||
+1
-1
@@ -20,7 +20,7 @@ import java.util.regex.Pattern
|
||||
|
||||
class JiangShuBlankDataStat {
|
||||
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new JiangShuBlankDataStat()
|
||||
|
||||
+1
-1
@@ -12,7 +12,7 @@ import com.mongodb.DBCollection
|
||||
|
||||
class TestArroundPrice {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection DIST_DATA_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_analy_data", lineUrl)
|
||||
|
||||
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
|
||||
|
||||
+1
-1
@@ -15,7 +15,7 @@ import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class StreetProcess {
|
||||
|
||||
final DBCollection TOWNSHIP = MongoUtils.getCollectionByName("fangjia", "township", "192.168.0.43")
|
||||
final DBCollection TOWNSHIP = MongoUtils.getCollectionByName("fangjia", "township", "mongo.db.fangjia.com")
|
||||
|
||||
public static final def MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
public static final DBCollection TOWNSHIP100 = MongoUtils.getCollectionByName2("fangjia", "township", MONGODB_URL)
|
||||
|
||||
+137
-6
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.gj.xiamen
|
||||
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import org.apache.http.HttpHost
|
||||
import org.apache.http.auth.AuthScope
|
||||
import org.apache.http.auth.UsernamePasswordCredentials
|
||||
@@ -7,8 +8,10 @@ import org.apache.http.client.CredentialsProvider
|
||||
import org.apache.http.client.config.RequestConfig
|
||||
import org.apache.http.impl.client.BasicCredentialsProvider
|
||||
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder
|
||||
import org.apache.solr.client.solrj.request.UpdateRequest
|
||||
import org.elasticsearch.action.ActionListener
|
||||
import org.elasticsearch.action.bulk.BackoffPolicy
|
||||
import org.elasticsearch.action.bulk.BulkItemResponse
|
||||
import org.elasticsearch.action.bulk.BulkProcessor
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
@@ -17,11 +20,13 @@ import org.elasticsearch.action.search.ClearScrollRequest
|
||||
import org.elasticsearch.action.search.SearchRequest
|
||||
import org.elasticsearch.action.search.SearchResponse
|
||||
import org.elasticsearch.action.search.SearchScrollRequest
|
||||
import org.elasticsearch.action.support.WriteRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.client.RestClient
|
||||
import org.elasticsearch.client.RestClientBuilder
|
||||
import org.elasticsearch.client.RestHighLevelClient
|
||||
import org.elasticsearch.common.unit.TimeValue
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.search.Scroll
|
||||
import org.elasticsearch.search.SearchHit
|
||||
@@ -160,8 +165,17 @@ class ElasticUtils89 {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterBulk(long executionId, BulkRequest request,
|
||||
BulkResponse response) {
|
||||
public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
|
||||
if (response.hasFailures()) {
|
||||
for (BulkItemResponse itemResponse : response.getItems()) {
|
||||
if (itemResponse.isFailed()) {
|
||||
// 处理失败项
|
||||
String failureMessage = itemResponse.getFailureMessage();
|
||||
println failureMessage
|
||||
// 日志记录、告警或重试
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -205,6 +219,7 @@ class ElasticUtils89 {
|
||||
}
|
||||
BulkProcessor bulkProcessor = createBulkProcessor(host, client)
|
||||
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
|
||||
|
||||
SearchRequest searchRequest = new SearchRequest(indexName);
|
||||
searchRequest.scroll(scroll);
|
||||
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||
@@ -221,7 +236,7 @@ class ElasticUtils89 {
|
||||
// executorService.submit(new Runnable() {
|
||||
// @Override
|
||||
// void run() {
|
||||
try{
|
||||
// try{
|
||||
|
||||
if(closure.getMaximumNumberOfParameters() == 1){
|
||||
def update = closure.call(hit.getSourceAsMap())
|
||||
@@ -232,14 +247,16 @@ class ElasticUtils89 {
|
||||
def update = closure.call(hit.getId(), hit.getSourceAsMap())
|
||||
if(update){
|
||||
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
|
||||
|
||||
}
|
||||
}
|
||||
}catch (Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
// }catch (Exception e){
|
||||
// e.printStackTrace()
|
||||
// }
|
||||
// }
|
||||
// })
|
||||
}
|
||||
|
||||
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
|
||||
scrollRequest.scroll(scroll);
|
||||
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
|
||||
@@ -256,6 +273,120 @@ class ElasticUtils89 {
|
||||
client.close()
|
||||
}
|
||||
|
||||
static queryUpdate(String host, String indexName, QueryBuilder qb, Closure closure){
|
||||
RestHighLevelClient client = null
|
||||
if(host == "192.168.0.89"){
|
||||
client = getClient(host);
|
||||
}else if(host == "192.168.0.52"){
|
||||
client = getClientOther(host);
|
||||
}
|
||||
BulkProcessor bulkProcessor = createBulkProcessor(host, client)
|
||||
|
||||
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(30))
|
||||
// 2. 初始化滚动查询
|
||||
SearchRequest searchRequest = new SearchRequest(indexName);
|
||||
searchRequest.scroll(scroll);
|
||||
|
||||
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||
searchSourceBuilder.query(qb);
|
||||
searchRequest.source(searchSourceBuilder);
|
||||
|
||||
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
|
||||
String scrollId = searchResponse.getScrollId();
|
||||
SearchHit[] hits = searchResponse.getHits().getHits();
|
||||
|
||||
// 3. 批量操作容器
|
||||
BulkRequest bulkRequest = new BulkRequest();
|
||||
|
||||
// 4. 处理滚动结果
|
||||
while (hits.length > 0){
|
||||
for (SearchHit hit : hits) {
|
||||
// 示例修改逻辑:当满足条件时更新文档
|
||||
if(closure.getMaximumNumberOfParameters() == 2){
|
||||
def update = closure.call(hit.getId(), hit.getSourceAsMap())
|
||||
if(update){
|
||||
bulkRequest.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
|
||||
}
|
||||
}
|
||||
// 批量提交控制
|
||||
if (bulkRequest.numberOfActions() >= 1) {
|
||||
bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
System.err.println("批量操作部分失败: " + bulkResponse.buildFailureMessage());
|
||||
// 这里可以添加重试逻辑
|
||||
}
|
||||
bulkRequest = new BulkRequest(); // 重置容器
|
||||
}
|
||||
}
|
||||
|
||||
// 5. 获取下一批滚动结果
|
||||
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId).scroll(TimeValue.timeValueMinutes(5));
|
||||
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
|
||||
scrollId = searchResponse.getScrollId();
|
||||
hits = searchResponse.getHits().getHits();
|
||||
}
|
||||
|
||||
// 6. 提交剩余批量操作
|
||||
if (bulkRequest.numberOfActions() > 0) {
|
||||
BulkResponse bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
System.err.println("批量操作部分失败: " + bulkResponse.buildFailureMessage());
|
||||
}
|
||||
}
|
||||
|
||||
// 7. 清理滚动上下文
|
||||
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
|
||||
clearScrollRequest.addScrollId(scrollId);
|
||||
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
|
||||
|
||||
bulkProcessor.close()
|
||||
bulkProcessor.awaitClose(1, TimeUnit.DAYS)
|
||||
client.close()
|
||||
}
|
||||
|
||||
|
||||
static updateEs(addEsDatasMap, host, indexName){
|
||||
BulkRequest request = new BulkRequest();
|
||||
int processedCount = 0;
|
||||
def client = null
|
||||
if(host && host == "192.168.0.89"){
|
||||
client = ElasticUtils89.getClient(host)
|
||||
}else if(host && host == "192.168.0.52"){
|
||||
client = ElasticUtils.getClient(host)
|
||||
}
|
||||
|
||||
addEsDatasMap.each{id, map->
|
||||
request.add(new IndexRequest(indexName, "_doc", id).source(map, XContentType.JSON));
|
||||
if (request.numberOfActions() >= 100) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents "+host
|
||||
}
|
||||
request = new BulkRequest();
|
||||
}
|
||||
}
|
||||
|
||||
if (request.numberOfActions() > 0) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents "+host
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
static def executeSimpleQuery(String host, String indexName, QueryBuilder qb, includes){
|
||||
|
||||
|
||||
+2
-2
@@ -27,9 +27,9 @@ class NetworkDistanceBatch extends BaseBatch{
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
|
||||
final DBCollection NETWORK_COLL = MongoUtils.getCollectionByName("business", "xm_network", "192.168.0.43")
|
||||
final DBCollection NETWORK_COLL = MongoUtils.getCollectionByName("business", "xm_network", "mongo.db.fangjia.com")
|
||||
|
||||
final DBCollection MAP_NETWORK_COLL = MongoUtils.getCollectionByName("business", "xm_network_map", "192.168.0.43")
|
||||
final DBCollection MAP_NETWORK_COLL = MongoUtils.getCollectionByName("business", "xm_network_map", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new NetworkDistanceBatch(), args)
|
||||
|
||||
+3
-4
@@ -78,8 +78,6 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
|
||||
run()
|
||||
// println "全国district, shop, office同步完成"
|
||||
//民政部编码上传
|
||||
@@ -167,9 +165,9 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
|
||||
def shops = updating(c, "shop")
|
||||
updateEs(c, shops, "shop", "192.168.0.89")
|
||||
updateEs(c, offices, "shop", "192.168.0.52")
|
||||
updateEs(c, shops, "shop", "192.168.0.52")
|
||||
|
||||
println index +" "+ c + " 完成"
|
||||
println index +" "+ c + " 完成 "+new Date().toString()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -245,6 +243,7 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
}
|
||||
if(!esObj || (esObj && diffMap)){
|
||||
map.put("id", d._id.toString())
|
||||
map.put("status", 0) //因为判断是否有值,0会返回false,所在这里设置默认值,在89和52上的都是有效数据
|
||||
return map
|
||||
}
|
||||
return null
|
||||
|
||||
+2
-2
@@ -20,10 +20,10 @@ import java.util.regex.Pattern
|
||||
class ConsumptionRatingBatch extends BaseBatch {
|
||||
def weekend = DateUtil.truncateDay(DateUtil.getFirstDayOfWeek(new Date()))
|
||||
public static
|
||||
final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43")
|
||||
final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
public static
|
||||
final DBCollection CONSUMPTION_GRADE_COLL = MongoUtils.getCollectionByName("fangjia_stat", "consumption_grade", "192.168.0.85")
|
||||
final DBCollection CONSUMPTION_GRADE_COLL = MongoUtils.getCollectionByName("fangjia_stat", "consumption_grade", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
final DBCollection AMAP_POI_New_COLL = MongoUtils.getCollectionByName("m_dianping", "dianping_9", "114.80.150.196", 27777)
|
||||
|
||||
+1
-1
@@ -310,7 +310,7 @@ class SMSOutgoingListener extends BaseBatch {
|
||||
if(StringUtils.isNotBlank(checkCode)){
|
||||
JSONObject con = new JSONObject()
|
||||
con.put("code", checkCode);
|
||||
return AliyunSmsClient.sendSms(mobile, "SMS_482955117", con.toString())
|
||||
return AliyunSmsClient.sendSms(mobile, "SMS_483345375", con.toString())
|
||||
}
|
||||
}
|
||||
int resCode = SingletonClient.getClient().sendSMS([mobile] as String[],content, 5)
|
||||
|
||||
+1
-1
@@ -52,7 +52,7 @@ class AliyunSmsClient {
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
sendSms("13916076378", "SMS_135220034", '{"code":12345}')
|
||||
sendSms("15021384995", "SMS_482915107", '{"code":12345}')
|
||||
}
|
||||
|
||||
|
||||
|
||||
+1
-1
@@ -30,7 +30,7 @@ import java.sql.SQLException
|
||||
class HousingTransactionExportBatch extends BaseBatch {
|
||||
|
||||
|
||||
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
|
||||
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
static DBCollection _db_esf_transaction = MongoUtils.getCollectionByName("vip_site", "esf_transaction", "192.168.0.235", 27018);
|
||||
static MongoCollection<Document> PAI_MAI = Mongo3Utils.getDbCollection(Constants.FANGJIA_MONGO_URI, "fangjia_crawl", "pai_mai")
|
||||
|
||||
|
||||
+1
-1
@@ -23,7 +23,7 @@ import java.util.regex.Pattern
|
||||
|
||||
class CqfdcMatch {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
|
||||
public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", lineUrl)
|
||||
|
||||
static DBCollection loudongFdcConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong_cqfdc", "192.168.0.235", 27018)
|
||||
|
||||
+2
-2
@@ -53,8 +53,8 @@ def pcmap = [:]
|
||||
provinceList.remove("广西省")
|
||||
provinceList.add("广西壮族自治区")
|
||||
|
||||
DSCollection seabird = new DataStore("192.168.0.43", 28017).getCollection("fangjia_base.baidu_place_poi2")
|
||||
DBCollection m43 = MongoUtils.getCollectionByName("fangjia_base", "baidu_place_poi2", "192.168.0.43", 28017)
|
||||
DSCollection seabird = new DataStore("mongo.db.fangjia.com", 28017).getCollection("fangjia_base.baidu_place_poi2")
|
||||
DBCollection m43 = MongoUtils.getCollectionByName("fangjia_base", "baidu_place_poi2", "mongo.db.fangjia.com", 28017)
|
||||
//def type3Not = ["学校内部设施", "进港", "出港", "候船室", "住宿服务相关"]
|
||||
//
|
||||
//def type2Not = ["学校", "专科医院", "综合医院", "诊所", "银行", "自动提款机", "便民商店/便利店", "家电电子卖场", "超级市场", "商场", "美容美发店", "地铁站", "港口码头", "住宿服务"]
|
||||
|
||||
+7
-7
@@ -16,9 +16,9 @@ import com.mongodb.DBCollection
|
||||
//@BatchOwner(Watcher.MaJinTao)
|
||||
class CommunityHouseSampleBatch extends BaseBatch {
|
||||
|
||||
DBCollection SEAWEED = MongoUtils.getCollectionByName("fangjia", "seaweed", "192.168.0.43", 28017)
|
||||
DBCollection STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 28017)
|
||||
DBCollection HOUSE_CASE = MongoUtils.getCollectionByName("business", "house_case", "192.168.0.43", 28017)
|
||||
DBCollection SEAWEED = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com", 28017)
|
||||
DBCollection STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com", 28017)
|
||||
DBCollection HOUSE_CASE = MongoUtils.getCollectionByName("business", "house_case", "mongo.db.fangjia.com", 28017)
|
||||
DSCollection SEABIRD = new DataStore("seabird.db.fangjia.com").getCollection("fangjia.seabird")
|
||||
|
||||
static void main(String[] args) {
|
||||
@@ -160,13 +160,13 @@ import com.mongodb.DBCollection
|
||||
@BatchOwner(Watcher.MaJinTao)
|
||||
class CommunityHouseSampleBatch extends BaseBatch {
|
||||
|
||||
DBCollection SEAWEED = MongoUtils.getCollectionByName("fangjia", "seaweed", "192.168.0.43", 28017)
|
||||
DBCollection SEAWEED = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com", 28017)
|
||||
|
||||
DBCollection STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 28017)
|
||||
DBCollection STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com", 28017)
|
||||
|
||||
DBCollection HOUSE_CASE = MongoUtils.getCollectionByName("business", "house_case", "192.168.0.43", 28017)
|
||||
DBCollection HOUSE_CASE = MongoUtils.getCollectionByName("business", "house_case", "mongo.db.fangjia.com", 28017)
|
||||
|
||||
// DBCollection SEABIRD = MongoUtils.getCollectionByName("fangjia", "seabird", "192.168.0.43", 28017)
|
||||
// DBCollection SEABIRD = MongoUtils.getCollectionByName("fangjia", "seabird", "1mongo.db.fangjia.com", 28017)
|
||||
|
||||
DSCollection SEABIRD = new DataStore("seabird.db.fangjia.com").getCollection("fangjia.seabird")
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user