Merge remote-tracking branch 'origin/master'

This commit is contained in:
majintao
2025-04-15 10:31:12 +08:00
149 changed files with 3160 additions and 341 deletions
@@ -0,0 +1,9 @@
package com.fangjia.dal.mongo.dao.shop;
import com.fangjia.dal.MongoDao;
import com.fangjia.dal.mongo.model.ShopRentStat;
public interface ShopRentStatDao extends MongoDao<ShopRentStat> {
}
@@ -0,0 +1,8 @@
package com.fangjia.dal.mongo.dao.shop;
import com.fangjia.dal.MongoDaoImpl;
import com.fangjia.dal.mongo.model.ShopRentStat;
public class ShopRentStatDaoImpl extends MongoDaoImpl<ShopRentStat> implements ShopRentStatDao {
}
@@ -0,0 +1,9 @@
package com.fangjia.dal.mongo.dao.shop;
import com.fangjia.dal.MongoDao;
import com.fangjia.dal.mongo.model.ShopSellStat;
public interface ShopSellStatDao extends MongoDao<ShopSellStat> {
}
@@ -0,0 +1,8 @@
package com.fangjia.dal.mongo.dao.shop;
import com.fangjia.dal.MongoDaoImpl;
import com.fangjia.dal.mongo.model.ShopSellStat;
public class ShopSellStatDaoImpl extends MongoDaoImpl<ShopSellStat> implements ShopSellStatDao {
}
@@ -0,0 +1,84 @@
package com.fangjia.dal.mongo.model;
import com.fangjia.dal.MongoEntity;
import javax.persistence.*;
@Entity
@Table(name = "fangjia_stat.shop_rent_stat")
public class ShopRentStat implements MongoEntity {
private static final long serialVersionUID = 4425229045088250573L;
private String id;
private String cat;
private String city;
private String district;
private Integer date;
private Double avgPrice;
private String name;
@Id
@GeneratedValue
@Column(name = "_id", unique = true, nullable = false)
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
@Column(name = "cat")
public String getCat() {
return cat;
}
public void setCat(String cat) {
this.cat = cat;
}
@Column(name = "city")
public String getCity() {
return city;
}
public void setCity(String city) {
this.city = city;
}
@Column(name = "district")
public String getDistrict() {
return district;
}
public void setDistrict(String district) {
this.district = district;
}
@Column(name = "date")
public Integer getDate() {
return date;
}
public void setDate(Integer date) {
this.date = date;
}
@Column(name = "avg_price")
public Double getAvgPrice() {
return avgPrice;
}
public void setAvgPrice(Double avgPrice) {
this.avgPrice = avgPrice;
}
@Column(name = "name")
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
}
@@ -0,0 +1,84 @@
package com.fangjia.dal.mongo.model;
import com.fangjia.dal.MongoEntity;
import javax.persistence.*;
@Entity
@Table(name = "fangjia_stat.shop_sell_stat")
public class ShopSellStat implements MongoEntity {
private static final long serialVersionUID = 4425229045088250573L;
private String id;
private String cat;
private String city;
private String district;
private Integer date;
private Integer avgPrice;
private String name;
@Id
@GeneratedValue
@Column(name = "_id", unique = true, nullable = false)
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
@Column(name = "cat")
public String getCat() {
return cat;
}
public void setCat(String cat) {
this.cat = cat;
}
@Column(name = "city")
public String getCity() {
return city;
}
public void setCity(String city) {
this.city = city;
}
@Column(name = "district")
public String getDistrict() {
return district;
}
public void setDistrict(String district) {
this.district = district;
}
@Column(name = "date")
public Integer getDate() {
return date;
}
public void setDate(Integer date) {
this.date = date;
}
@Column(name = "avg_price")
public Integer getAvgPrice() {
return avgPrice;
}
public void setAvgPrice(Integer avgPrice) {
this.avgPrice = avgPrice;
}
@Column(name = "name")
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
}
@@ -0,0 +1,614 @@
package com.fangjia.open.controllers;
import com.fangjia.core.RegionData;
import com.fangjia.core.util.AnalyticsUtil;
import com.fangjia.core.util.DataUtil;
import com.fangjia.core.util.HttpRequest;
import com.fangjia.core.util.LoggerFactory;
import com.fangjia.dal.common.ServerAreaData;
import com.fangjia.dal.mongo.fangjia.model.ServerArea;
import com.fangjia.dal.mongo.model.Evaluation;
import com.fangjia.dal.mongo.model.ShopRentStat;
import com.fangjia.open.HttpCode;
import com.fangjia.open.OpenHttpCodeEnum;
import com.fangjia.open.interceptors.AuthenticationBean;
import com.fangjia.open.interceptors.Authority;
import com.fangjia.open.interceptors.Role;
import com.fangjia.open.model.BasicModel;
import com.fangjia.open.model.PropertyEvaluationModel;
import com.fangjia.open.model.ReportApplyModel;
import com.fangjia.open.model.RestfulModelAndView;
import com.fangjia.open.util.CommonUtils;
import com.fangjia.open.util.FangJiaTongUrl;
import com.fangjia.service.address.AddressParser;
import com.fangjia.service.datastore.FieldFilterService;
import com.fangjia.service.datastore.AiShopService;
import com.fangjia.service.datastore.OpenApiLogService;
import com.fangjia.service.datastore.dto.ShopEvaluationDto;
import net.sf.json.JSONObject;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.sothis.web.mvc.ActionContext;
import org.sothis.web.mvc.annotation.Param;
import javax.servlet.http.HttpServletRequest;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static com.fangjia.open.util.CommonUtils.convertToString;
import static com.fangjia.open.util.EvaluationFactorUtil.*;
/**
* 商铺出租服务类
*
* @author shenjian
* @since 2025/4/10
*/
@Authority(role = Role.USER)
public class ShopController {
private static final Logger LOGGER = LoggerFactory.getLogger(PropertyController.class);
private AiShopService aiShopService;
private OpenApiLogService openApiLogService;
private AddressParser addressParser;
private FieldFilterService fieldFilterService;
/**
* 城市出租均价
*
* @return
*/
public Object cityRentAction(@Param(name = "city") String city, @Param(name = "startDate") Integer startDate, @Param(name = "endDate") Integer endDate) {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
if (StringUtils.isBlank(city)) {
mav.setModel(BasicModel.buildBadRequestModel("city不能为null"));
return mav;
}
if (!RegionData.isCity(city)) {
String formatCity = RegionData.formatCity(city);
if (StringUtils.isNotBlank(formatCity)) {
city = formatCity;
} else {
mav.setModel(BasicModel.buildBadRequestModel("city非法"));
return mav;
}
}
ServerArea serverArea = ServerAreaData.getServerAreaByCity(city);
if (null == serverArea) {
mav.setModel(BasicModel.buildBadRequestModel("city非法"));
return mav;
}
if (startDate == null || endDate == null) {
mav.setModel(BasicModel.buildBadRequestModel("startDate或endDate不能为null"));
return mav;
}
if (startDate > endDate) {
mav.setModel(BasicModel.buildBadRequestModel("startDate不能大于endDate"));
return mav;
}
List<ShopRentStat> regionStats = aiShopService.findShopRentStat(city, null, null, "city", startDate, endDate);
if (CollectionUtils.isEmpty(regionStats)) {
mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "该城市没有均价数据"));
return mav;
}
mav.setModel(BasicModel.buildOKModel().setResult(regionStats));
return mav;
}
/**
* 区域出租均价
*
* @return
*/
public Object districtRentAction(@Param(name = "city") String city, @Param(name = "district") String district,
@Param(name = "startDate") Integer startDate, @Param(name = "endDate") Integer endDate) {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
if (StringUtils.isBlank(city)) {
mav.setModel(BasicModel.buildBadRequestModel("city不能为null"));
return mav;
}
if (StringUtils.isBlank(district)) {
mav.setModel(BasicModel.buildBadRequestModel("district不能为null"));
return mav;
}
if (!RegionData.isCity(city)) {
String formatCity = RegionData.formatCity(city);
if (StringUtils.isNotBlank(formatCity)) {
city = formatCity;
} else {
mav.setModel(BasicModel.buildBadRequestModel("city非法"));
return mav;
}
}
ServerArea serverArea = ServerAreaData.getServerAreaByCity(city);
if (null == serverArea) {
mav.setModel(BasicModel.buildBadRequestModel("city非法"));
return mav;
}
String region = RegionData.formatRegion(city, district);
if (!RegionData.isRegion(city, region)) {
mav.setModel(BasicModel.buildBadRequestModel("没有找你输入的区域"));
return mav;
}
if (startDate == null || endDate == null) {
mav.setModel(BasicModel.buildBadRequestModel("startDate或endDate不能为null"));
return mav;
}
if (startDate > endDate) {
mav.setModel(BasicModel.buildBadRequestModel("startDate不能大于endDate"));
return mav;
}
List<ShopRentStat> regionStats = aiShopService.findShopRentStat(city, region, null, "district", startDate, endDate);
if (CollectionUtils.isEmpty(regionStats)) {
mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "该区域没有均价数据"));
return mav;
}
mav.setModel(BasicModel.buildOKModel().setResult(regionStats));
return mav;
}
/**
* 写字楼均价走势
*
* @return
*/
public Object shopRentAction(@Param(name = "city") String city, @Param(name = "district") String district,
@Param(name = "name") String name, @Param(name = "startDate") Integer startDate,
@Param(name = "endDate") Integer endDate) {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
if (StringUtils.isBlank(city)) {
mav.setModel(BasicModel.buildBadRequestModel("city不能为null"));
return mav;
}
if (StringUtils.isBlank(district)) {
mav.setModel(BasicModel.buildBadRequestModel("district不能为null"));
return mav;
}
if (!RegionData.isCity(city)) {
String formatCity = RegionData.formatCity(city);
if (StringUtils.isNotBlank(formatCity)) {
city = formatCity;
} else {
mav.setModel(BasicModel.buildBadRequestModel("city非法"));
return mav;
}
}
ServerArea serverArea = ServerAreaData.getServerAreaByCity(city);
if (null == serverArea) {
mav.setModel(BasicModel.buildBadRequestModel("city非法"));
return mav;
}
String region = RegionData.formatRegion(city, district);
if (!RegionData.isRegion(city, region)) {
mav.setModel(BasicModel.buildBadRequestModel("没有找你输入的区域"));
return mav;
}
if (startDate == null || endDate == null) {
mav.setModel(BasicModel.buildBadRequestModel("startDate或endDate不能为null"));
return mav;
}
if (startDate > endDate) {
mav.setModel(BasicModel.buildBadRequestModel("startDate不能大于endDate"));
return mav;
}
List<ShopRentStat> regionStats = aiShopService.findShopRentStat(city, region, name, "shop", startDate, endDate);
if (CollectionUtils.isEmpty(regionStats)) {
mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "该写字楼没有均价数据"));
return mav;
}
mav.setModel(BasicModel.buildOKModel().setResult(regionStats));
return mav;
}
/**
* 写字楼楼租金地址估值
*
* @param model
* @param request
* @return
*/
public Object addressEvaluateRentAction(PropertyEvaluationModel model, HttpServletRequest request) {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
if (StringUtils.isBlank(model.getCity()) || StringUtils.isBlank(model.getAddress()) || model.getSize() <= 0) {
mav.setModel(BasicModel.buildBadRequestModel("参数非法:城市,地址,面积(size)为必填项"));
return mav;
}
if (model.getCity().endsWith("市")) {
model.setCity(model.getCity().replaceAll("市$", ""));
}
if (model.getCarportPrice() != null && (model.getCarportPrice() < 0 || model.getCarportPrice() > 1000000)) {
mav.setModel(BasicModel.buildBadRequestModel("carportPrice(车位)取值不合法"));
return mav;
}
if (model.getMaxFloor() != null && (model.getMaxFloor() <= 0 || model.getMaxFloor() > 59)) {
mav.setModel(BasicModel.buildBadRequestModel("maxFloor取值非法,正确范围[1,60)"));
return mav;
}
if (model.getFloor() != null && (model.getFloor() <= 0 || model.getFloor() > 59)) {
mav.setModel(BasicModel.buildBadRequestModel("floor取值非法,正确范围[1,60)"));
return mav;
}
if (model.getMaxFloor() != null && model.getFloor() != null && model.getFloor() > model.getMaxFloor()) {
mav.setModel(BasicModel.buildBadRequestModel("所在楼层不能大于总楼层"));
return mav;
}
ShopEvaluationDto evaluation = aiShopService.addressEvaluateRent(buildEvaluation(model, request));
if (evaluation.getUnitPrice() != null && evaluation.getUnitPrice() != 0) {
Map<String, Object> districtPriceMap = new HashMap<String, Object>();
districtPriceMap.put("unitPrice", evaluation.getUnitPrice());
districtPriceMap.put("city", evaluation.getCity());
districtPriceMap.put("district", evaluation.getDistrict());
districtPriceMap.put("name", evaluation.getName());
districtPriceMap.put("samples", evaluation.getSeawater());
districtPriceMap.put("location", evaluation.getLocation());
mav.setModel(BasicModel.buildOKModel().setResult(districtPriceMap));
try {
if (StringUtils.isNotBlank(evaluation.getFromUserId()) && !evaluation.getFromUserId().equals("1007497")) {
openApiLogService.saveLog(CommonUtils.convertToMap(model), districtPriceMap, evaluation, "/shop/addressEvaluateRent");
}
} catch (Exception e) {
e.printStackTrace();
}
return mav;
} else {
mav.setModel(BasicModel.buildOKModel().setMsg("暂时不支持该地址写字楼的评估").markNoData());
return mav;
}
}
/**
* 申请估值报告
*
* @param model 申请报告请求参数
* @return
*/
public Object assessPriceReportApplyAction(ReportApplyModel model) {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
if (StringUtils.isBlank(model.getClient())) {
mav.setModel(BasicModel.buildBadRequestModel("参数非法:委托机构为必填项"));
return mav;
}
if (StringUtils.isBlank(model.getSubjectMatter())) {
mav.setModel(BasicModel.buildBadRequestModel("参数非法:标的物为必填项"));
return mav;
}
if (StringUtils.isBlank(model.getCity()) || model.getCity().contains("全国")) {
mav.setModel(BasicModel.buildBadRequestModel("参数非法:城市为必填项且不能为全国"));
return mav;
}
if (StringUtils.isBlank(model.getDistrict()) || model.getDistrict().contains("全国")) {
mav.setModel(BasicModel.buildBadRequestModel("参数非法:区域为必填项且不能为全国"));
return mav;
}
if (model.getSize() == null || model.getSize() <= 0 ) {
mav.setModel(BasicModel.buildBadRequestModel("参数非法:面积(size)为必填项且大于0"));
return mav;
}
if (model.getCity().endsWith("市")) {
model.setCity(model.getCity().replaceAll("市$", ""));
}
if (model.getDistrict().endsWith("区")) {
model.setDistrict(model.getDistrict().replaceAll("区$", ""));
}
if (model.getMaxFloor() != null && (model.getMaxFloor() <= 0 || model.getMaxFloor() > 59)) {
mav.setModel(BasicModel.buildBadRequestModel("maxFloor取值非法,正确范围[1,60)"));
return mav;
}
if (model.getFloor() != null && (model.getFloor() <= 0 || model.getFloor() > 59)) {
mav.setModel(BasicModel.buildBadRequestModel("floor取值非法,正确范围[1,60)"));
return mav;
}
if (model.getMaxFloor() != null && model.getFloor() != null && model.getFloor() > model.getMaxFloor()) {
mav.setModel(BasicModel.buildBadRequestModel("所在楼层不能大于总楼层"));
return mav;
}
HttpRequest request = new HttpRequest();
JSONObject json = null;
Map<String, String> params = new HashMap<String, String>();
params.put("client", model.getClient());
params.put("subjectMatter", model.getSubjectMatter());
params.put("city", model.getCity());
params.put("address", StringUtils.isNotBlank(model.getAddress()) ? model.getAddress() : null);
params.put("district", model.getDistrict());
params.put("name", StringUtils.isNotBlank(model.getName()) ? model.getName() : null);
params.put("houseNum", StringUtils.isNotBlank(model.getBuildingNumber()) ? model.getBuildingNumber() : null);
params.put("roomNum", StringUtils.isNotBlank(model.getUnitNumber()) ? model.getUnitNumber() : null);
params.put("size", String.valueOf(model.getSize()));
params.put("houseFloor", model.getFloor() != null ? String.valueOf(model.getFloor()) : null);
params.put("houseHeight", model.getMaxFloor() != null ? String.valueOf(model.getMaxFloor()) : null);
params.put("buildType", StringUtils.isNotBlank(model.getPropertyType()) ? model.getPropertyType() : "0");
params.put("direction", StringUtils.isNotBlank(model.getDirection()) ? model.getDirection() : "3");
params.put("fitment", StringUtils.isNotBlank(model.getDecoration()) ? model.getDecoration() : "2");
JSONObject jsonObject = JSONObject.fromObject(params);
Map<String, String> headers = new HashMap<String, String>();
headers.put("Authorization", "Bearer " + FangJiaTongUrl.FANG_JIA_TONG_TOKEN);
headers.put("Content-Type", "application/json");
try {
String ret = request.postWithBody(FangJiaTongUrl.FANG_JIA_TONG_URL_UAT + FangJiaTongUrl.ASSESS_PRICE_REPORT_APPLY, jsonObject, headers, "UTF-8");
json = JSONObject.fromObject(FangJiaTongUrl.replaceAll(ret));
} catch (Exception e) {
try {
String ret = request.postWithBody(FangJiaTongUrl.FANG_JIA_TONG_URL_UAT + FangJiaTongUrl.ASSESS_PRICE_REPORT_APPLY, jsonObject, headers, "UTF-8");
json = JSONObject.fromObject(FangJiaTongUrl.replaceAll(ret));
} catch (Exception ex) {
ex.printStackTrace();
}
}
if (json != null) {
if (json.getInt("code") == 200 && json.containsKey("data")) {
HashMap<String, String> result = new HashMap<String, String>();
result.put("caseId", String.valueOf(json.get("data")));
mav.setModel(BasicModel.buildOKModel().setResult(result));
} else {
mav.setModel(BasicModel.buildBadRequestModel("申请估值报告失败:" + json.getString("message")));
}
} else {
mav.setModel(BasicModel.buildBadRequestModel("申请估值报告失败:调用申请报告接口返回为空"));
}
return mav;
}
/**
* 获取估值报告
*
* @param caseId 估值报告ID
* @return
*/
public Object assessPriceReportFetchAction(@Param(name = "caseId") String caseId) {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
if (StringUtils.isBlank(caseId)) {
mav.setModel(BasicModel.buildBadRequestModel("参数非法:估值报告业务编号为必填项"));
return mav;
}
HttpRequest request = new HttpRequest();
JSONObject json = null;
Map<String, String> params = new HashMap<String, String>();
params.put("caseId", caseId);
JSONObject jsonObject = JSONObject.fromObject(params);
Map<String, String> headers = new HashMap<String, String>();
headers.put("Authorization", "Bearer " + FangJiaTongUrl.FANG_JIA_TONG_TOKEN);
headers.put("Content-Type", "application/json");
try {
String ret = request.postWithBody(FangJiaTongUrl.FANG_JIA_TONG_URL_UAT + FangJiaTongUrl.ASSESS_PRICE_REPORT_FETCH, jsonObject, headers, "UTF-8");
json = JSONObject.fromObject(FangJiaTongUrl.replaceAll(ret));
} catch (Exception e) {
try {
String ret = request.postWithBody(FangJiaTongUrl.FANG_JIA_TONG_URL_UAT + FangJiaTongUrl.ASSESS_PRICE_REPORT_FETCH, jsonObject, headers, "UTF-8");
json = JSONObject.fromObject(FangJiaTongUrl.replaceAll(ret));
} catch (Exception ex) {
ex.printStackTrace();
}
}
if (json != null) {
if (json.getInt("code") == 200 && json.containsKey("data")) {
JSONObject resultJson = json.getJSONObject("data");
mav.setModel(BasicModel.buildOKModel().setResult(resultJson));
} else {
mav.setModel(BasicModel.buildBadRequestModel("获取估值报告失败:" + json.getString("message")));
}
} else {
mav.setModel(BasicModel.buildBadRequestModel("申请报告失败:调用获取估值报告接口返回为空"));
}
return mav;
}
private Integer getFloor(String roomNumber) {
if (StringUtils.isNotBlank(roomNumber) && DataUtil.isNumber(roomNumber)) {
if (roomNumber.length() == 3) {
return Integer.valueOf(roomNumber.substring(0, 1));
} else if (roomNumber.length() == 4) {
return Integer.valueOf(roomNumber.substring(0, 2));
} else {
return null;
}
}
return null;
}
/**
* 通过EvaluationModel组装Evaluation实体
*
* @param model
* @param request
* @return
*/
private Evaluation buildEvaluation(PropertyEvaluationModel model, HttpServletRequest request) {
Evaluation evaluation = new Evaluation();
evaluation.setLevel(model.getLevel());
evaluation.setFitment(model.getFitment());
evaluation.setType(model.getType());
evaluation.setCity(model.getCity());
evaluation.setRegion(model.getRegion());
evaluation.setDistrictName(model.getName());
evaluation.setAddress(model.getAddress());
evaluation.setDirection(model.getDirection());
evaluation.setNoise(NOISE.get(model.getNoise()));
evaluation.setType(model.getType());
evaluation.setSight(SIGHT.get(model.getView()));
evaluation.setAtticArea(model.getAtticSize());
evaluation.setTerraceArea(model.getOuterSize());
evaluation.setBasementArea(model.getBasementSize());
evaluation.setGardenArea(model.getGardenSize());
String worryFacility = model.getNegativeFactor();
evaluation.setWorryFacility(NEGATIVE_FACTOR.get(worryFacility));
evaluation.setCreationDate(new Date());
evaluation.setModificationDate(new Date());
evaluation.setClientNo(model.getClientNo());
if (0 != model.getAge()) {
evaluation.setAge(model.getAge());
}
evaluation.setArea(model.getSize());
if (ArrayUtils.isNotEmpty(model.getBuildQuality())) {
evaluation.setBuildQuality(convertToString(model.getBuildQuality()));
}
if (StringUtils.isNotBlank(model.getBuildType())) {
evaluation.setBuildType(model.getBuildType());
}
if (StringUtils.isNotBlank(model.getCarport())) {
evaluation.setCarport(model.getCarport());
}
if (model.getDecorationAge() >= 1 && model.getDecorationAge() <= 5) {
evaluation.setFitmentAge(model.getDecorationAge());
}
evaluation.setCarportPrice(model.getCarportPrice());
evaluation.setCity(model.getCity());
evaluation.setDistrictName(model.getName());
if (model.getDecorationCost() != null && model.getDecorationCost() > 0) {
evaluation.setFitmentCharge(model.getDecorationCost());
}
if (0 != model.getHall()) {
evaluation.setHall(model.getHall());
}
if (null != model.getFloor()) {
evaluation.setFloor(model.getFloor());
}
if (model.getMaxFloor() != null && 0 != model.getMaxFloor()) {
evaluation.setHeight(model.getMaxFloor());
}
if (StringUtils.isNotBlank(model.getBuildingNumber())) {
evaluation.setHouseNum(model.getBuildingNumber());
}
if (StringUtils.isNotBlank(model.getHouseProperty())) {
evaluation.setHouseProperty(model.getHouseProperty());
}
if (StringUtils.isNotBlank(model.getInsidepos())) {
evaluation.setInsidepos(model.getInsidepos());
}
if (StringUtils.isNotBlank(model.getLightingVentilation())) {
evaluation.setLightingVentilation(model.getLightingVentilation());
}
if (StringUtils.isNotBlank(model.getOtherFactor())) {
evaluation.setOtherFactor(model.getOtherFactor());
}
evaluation.setRegion(model.getDistrict());
if (null != model.getRoom()) {
evaluation.setRoom(model.getRoom());
}
if (StringUtils.isNotBlank(model.getRoomNumber())) {
evaluation.setRoomNum(model.getRoomNumber());
}
if (StringUtils.isNotBlank(model.getSideset())) {
evaluation.setSideset(model.getSideset());
}
if (0 != model.getToilet()) {
evaluation.setToilet(model.getToilet());
}
if (model.getFacilities() != null) {
String facilities = model.getFacilities();
String[] fs = facilities.split(",");
for (int i = 0; i < fs.length; i++) {
fs[i] = FACILITIES.get(fs[i]);
}
evaluation.setFacilities(convertToString(fs));
}
if (StringUtils.isNotBlank(model.getFacade())) {
evaluation.setFacade(model.getFacade());
}
if (StringUtils.isNotBlank(model.getTraffice())) {
evaluation.setTraffice(model.getTraffice());
}
if (StringUtils.isNotBlank(model.getShape())) {
evaluation.setShape(model.getShape());
}
if (StringUtils.isNotBlank(model.getObstacle())) {
evaluation.setObstacle(model.getObstacle());
}
if (StringUtils.isNotBlank(model.getIndustry())) {
evaluation.setIndustry(model.getIndustry());
}
if (StringUtils.isNotBlank(model.getSupportDevice())) {
evaluation.setSupportDevice(model.getSupportDevice());
}
if (StringUtils.isNotBlank(model.getLeftPosition())) {
evaluation.setLeftPosition(model.getLeftPosition());
}
if (StringUtils.isNotBlank(model.getShopType())) {
evaluation.setShopType(model.getShopType());
}
if (StringUtils.isNotBlank(model.getShopUnique())) {
evaluation.setShopUnique(model.getShopUnique());
}
if (StringUtils.isNotBlank(model.getLocation())) {
evaluation.setLocation(model.getLocation());
}
if (model.isNewDistrict()) {
evaluation.setNewDistrict(model.isNewDistrict());
}
String floorDesc = "中";
if (evaluation.getFloor() != null && evaluation.getHeight() != null) {
Double floorInfo = evaluation.getFloor() * 1.0 / evaluation.getHeight();
if (floorInfo <= 0.33) {
floorDesc = "低";
} else if (floorInfo > 0.66) {
floorDesc = "高";
}
evaluation.setFloorDesc(floorDesc);
}
String ip = AnalyticsUtil.getIpAddress(ActionContext.getContext().getRequest());
evaluation.setIp(ip);
AuthenticationBean authenticationBean = (AuthenticationBean) request
.getAttribute(AuthenticationBean.ATTR_AUTHENTICATION_BEAN);
if (authenticationBean != null && authenticationBean.apiUser != null) {
evaluation.setFromUserId(String.valueOf(authenticationBean.apiUser.getId()));
}
evaluation.setSource("api_shop_evaluateRent");
return evaluation;
}
private static final Map<String, String> FACILITIES = new HashMap<String, String>() {
{
put("0", "健身房");
put("1", "游泳池");
put("2", "会所");
put("3", "网球场");
put("4", "球场");
put("5", "棋牌室");
put("6", "停车场");
put("7", "图书馆");
put("8", "活动中心");
}
};
public void setOpenApiLogService(OpenApiLogService openApiLogService) {
this.openApiLogService = openApiLogService;
}
public void setFieldFilterService(FieldFilterService fieldFilterService) {
this.fieldFilterService = fieldFilterService;
}
public void setAiShopService(AiShopService aiShopService) {
this.aiShopService = aiShopService;
}
}
@@ -0,0 +1,614 @@
package com.fangjia.open.controllers;
import com.fangjia.core.RegionData;
import com.fangjia.core.util.AnalyticsUtil;
import com.fangjia.core.util.DataUtil;
import com.fangjia.core.util.HttpRequest;
import com.fangjia.core.util.LoggerFactory;
import com.fangjia.dal.common.ServerAreaData;
import com.fangjia.dal.mongo.fangjia.model.ServerArea;
import com.fangjia.dal.mongo.model.Evaluation;
import com.fangjia.dal.mongo.model.ShopSellStat;
import com.fangjia.open.HttpCode;
import com.fangjia.open.OpenHttpCodeEnum;
import com.fangjia.open.interceptors.AuthenticationBean;
import com.fangjia.open.interceptors.Authority;
import com.fangjia.open.interceptors.Role;
import com.fangjia.open.model.BasicModel;
import com.fangjia.open.model.PropertyEvaluationModel;
import com.fangjia.open.model.ReportApplyModel;
import com.fangjia.open.model.RestfulModelAndView;
import com.fangjia.open.util.CommonUtils;
import com.fangjia.open.util.FangJiaTongUrl;
import com.fangjia.service.address.AddressParser;
import com.fangjia.service.datastore.AiShopService;
import com.fangjia.service.datastore.FieldFilterService;
import com.fangjia.service.datastore.OpenApiLogService;
import com.fangjia.service.datastore.dto.ShopEvaluationDto;
import net.sf.json.JSONObject;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.sothis.web.mvc.ActionContext;
import org.sothis.web.mvc.annotation.Param;
import javax.servlet.http.HttpServletRequest;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static com.fangjia.open.util.CommonUtils.convertToString;
import static com.fangjia.open.util.EvaluationFactorUtil.*;
/**
* 商铺出售服务类
*
* @author shenjian
* @since 2025/4/14
*/
@Authority(role = Role.USER)
public class ShopSellController {
private static final Logger LOGGER = LoggerFactory.getLogger(PropertyController.class);
private AiShopService aiShopService;
private OpenApiLogService openApiLogService;
private AddressParser addressParser;
private FieldFilterService fieldFilterService;
/**
* 城市出租均价
*
* @return
*/
public Object citySellAction(@Param(name = "city") String city, @Param(name = "startDate") Integer startDate, @Param(name = "endDate") Integer endDate) {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
if (StringUtils.isBlank(city)) {
mav.setModel(BasicModel.buildBadRequestModel("city不能为null"));
return mav;
}
if (!RegionData.isCity(city)) {
String formatCity = RegionData.formatCity(city);
if (StringUtils.isNotBlank(formatCity)) {
city = formatCity;
} else {
mav.setModel(BasicModel.buildBadRequestModel("city非法"));
return mav;
}
}
ServerArea serverArea = ServerAreaData.getServerAreaByCity(city);
if (null == serverArea) {
mav.setModel(BasicModel.buildBadRequestModel("city非法"));
return mav;
}
if (startDate == null || endDate == null) {
mav.setModel(BasicModel.buildBadRequestModel("startDate或endDate不能为null"));
return mav;
}
if (startDate > endDate) {
mav.setModel(BasicModel.buildBadRequestModel("startDate不能大于endDate"));
return mav;
}
List<ShopSellStat> regionStats = aiShopService.findShopSellStat(city, null, null, "city", startDate, endDate);
if (CollectionUtils.isEmpty(regionStats)) {
mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "该城市没有均价数据"));
return mav;
}
mav.setModel(BasicModel.buildOKModel().setResult(regionStats));
return mav;
}
/**
* 区域出租均价
*
* @return
*/
public Object districtSellAction(@Param(name = "city") String city, @Param(name = "district") String district,
@Param(name = "startDate") Integer startDate, @Param(name = "endDate") Integer endDate) {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
if (StringUtils.isBlank(city)) {
mav.setModel(BasicModel.buildBadRequestModel("city不能为null"));
return mav;
}
if (StringUtils.isBlank(district)) {
mav.setModel(BasicModel.buildBadRequestModel("district不能为null"));
return mav;
}
if (!RegionData.isCity(city)) {
String formatCity = RegionData.formatCity(city);
if (StringUtils.isNotBlank(formatCity)) {
city = formatCity;
} else {
mav.setModel(BasicModel.buildBadRequestModel("city非法"));
return mav;
}
}
ServerArea serverArea = ServerAreaData.getServerAreaByCity(city);
if (null == serverArea) {
mav.setModel(BasicModel.buildBadRequestModel("city非法"));
return mav;
}
String region = RegionData.formatRegion(city, district);
if (!RegionData.isRegion(city, region)) {
mav.setModel(BasicModel.buildBadRequestModel("没有找你输入的区域"));
return mav;
}
if (startDate == null || endDate == null) {
mav.setModel(BasicModel.buildBadRequestModel("startDate或endDate不能为null"));
return mav;
}
if (startDate > endDate) {
mav.setModel(BasicModel.buildBadRequestModel("startDate不能大于endDate"));
return mav;
}
List<ShopSellStat> regionStats = aiShopService.findShopSellStat(city, region, null, "district", startDate, endDate);
if (CollectionUtils.isEmpty(regionStats)) {
mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "该区域没有均价数据"));
return mav;
}
mav.setModel(BasicModel.buildOKModel().setResult(regionStats));
return mav;
}
/**
* 写字楼均价走势
*
* @return
*/
public Object shopSellAction(@Param(name = "city") String city, @Param(name = "district") String district,
@Param(name = "name") String name, @Param(name = "startDate") Integer startDate,
@Param(name = "endDate") Integer endDate) {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
if (StringUtils.isBlank(city)) {
mav.setModel(BasicModel.buildBadRequestModel("city不能为null"));
return mav;
}
if (StringUtils.isBlank(district)) {
mav.setModel(BasicModel.buildBadRequestModel("district不能为null"));
return mav;
}
if (!RegionData.isCity(city)) {
String formatCity = RegionData.formatCity(city);
if (StringUtils.isNotBlank(formatCity)) {
city = formatCity;
} else {
mav.setModel(BasicModel.buildBadRequestModel("city非法"));
return mav;
}
}
ServerArea serverArea = ServerAreaData.getServerAreaByCity(city);
if (null == serverArea) {
mav.setModel(BasicModel.buildBadRequestModel("city非法"));
return mav;
}
String region = RegionData.formatRegion(city, district);
if (!RegionData.isRegion(city, region)) {
mav.setModel(BasicModel.buildBadRequestModel("没有找你输入的区域"));
return mav;
}
if (startDate == null || endDate == null) {
mav.setModel(BasicModel.buildBadRequestModel("startDate或endDate不能为null"));
return mav;
}
if (startDate > endDate) {
mav.setModel(BasicModel.buildBadRequestModel("startDate不能大于endDate"));
return mav;
}
List<ShopSellStat> regionStats = aiShopService.findShopSellStat(city, region, name, "shop", startDate, endDate);
if (CollectionUtils.isEmpty(regionStats)) {
mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "该写字楼没有均价数据"));
return mav;
}
mav.setModel(BasicModel.buildOKModel().setResult(regionStats));
return mav;
}
/**
* 写字楼楼租金地址估值
*
* @param model
* @param request
* @return
*/
public Object addressEvaluateSellAction(PropertyEvaluationModel model, HttpServletRequest request) {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
if (StringUtils.isBlank(model.getCity()) || StringUtils.isBlank(model.getAddress()) || model.getSize() <= 0) {
mav.setModel(BasicModel.buildBadRequestModel("参数非法:城市,地址,面积(size)为必填项"));
return mav;
}
if (model.getCity().endsWith("市")) {
model.setCity(model.getCity().replaceAll("市$", ""));
}
if (model.getCarportPrice() != null && (model.getCarportPrice() < 0 || model.getCarportPrice() > 1000000)) {
mav.setModel(BasicModel.buildBadRequestModel("carportPrice(车位)取值不合法"));
return mav;
}
if (model.getMaxFloor() != null && (model.getMaxFloor() <= 0 || model.getMaxFloor() > 59)) {
mav.setModel(BasicModel.buildBadRequestModel("maxFloor取值非法,正确范围[1,60)"));
return mav;
}
if (model.getFloor() != null && (model.getFloor() <= 0 || model.getFloor() > 59)) {
mav.setModel(BasicModel.buildBadRequestModel("floor取值非法,正确范围[1,60)"));
return mav;
}
if (model.getMaxFloor() != null && model.getFloor() != null && model.getFloor() > model.getMaxFloor()) {
mav.setModel(BasicModel.buildBadRequestModel("所在楼层不能大于总楼层"));
return mav;
}
ShopEvaluationDto evaluation = aiShopService.addressEvaluateSell(buildEvaluation(model, request));
if (evaluation.getUnitPrice() != null && evaluation.getUnitPrice() != 0) {
Map<String, Object> districtPriceMap = new HashMap<String, Object>();
districtPriceMap.put("unitPrice", evaluation.getUnitPrice());
districtPriceMap.put("city", evaluation.getCity());
districtPriceMap.put("district", evaluation.getDistrict());
districtPriceMap.put("name", evaluation.getName());
districtPriceMap.put("samples", evaluation.getSeawater());
districtPriceMap.put("location", evaluation.getLocation());
mav.setModel(BasicModel.buildOKModel().setResult(districtPriceMap));
try {
if (StringUtils.isNotBlank(evaluation.getFromUserId()) && !evaluation.getFromUserId().equals("1007497")) {
openApiLogService.saveLog(CommonUtils.convertToMap(model), districtPriceMap, evaluation, "/shop/addressEvaluateSell");
}
} catch (Exception e) {
e.printStackTrace();
}
return mav;
} else {
mav.setModel(BasicModel.buildOKModel().setMsg("暂时不支持该地址写字楼的评估").markNoData());
return mav;
}
}
/**
* 申请估值报告
*
* @param model 申请报告请求参数
* @return
*/
public Object assessPriceReportApplyAction(ReportApplyModel model) {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
if (StringUtils.isBlank(model.getClient())) {
mav.setModel(BasicModel.buildBadRequestModel("参数非法:委托机构为必填项"));
return mav;
}
if (StringUtils.isBlank(model.getSubjectMatter())) {
mav.setModel(BasicModel.buildBadRequestModel("参数非法:标的物为必填项"));
return mav;
}
if (StringUtils.isBlank(model.getCity()) || model.getCity().contains("全国")) {
mav.setModel(BasicModel.buildBadRequestModel("参数非法:城市为必填项且不能为全国"));
return mav;
}
if (StringUtils.isBlank(model.getDistrict()) || model.getDistrict().contains("全国")) {
mav.setModel(BasicModel.buildBadRequestModel("参数非法:区域为必填项且不能为全国"));
return mav;
}
if (model.getSize() == null || model.getSize() <= 0 ) {
mav.setModel(BasicModel.buildBadRequestModel("参数非法:面积(size)为必填项且大于0"));
return mav;
}
if (model.getCity().endsWith("市")) {
model.setCity(model.getCity().replaceAll("市$", ""));
}
if (model.getDistrict().endsWith("区")) {
model.setDistrict(model.getDistrict().replaceAll("区$", ""));
}
if (model.getMaxFloor() != null && (model.getMaxFloor() <= 0 || model.getMaxFloor() > 59)) {
mav.setModel(BasicModel.buildBadRequestModel("maxFloor取值非法,正确范围[1,60)"));
return mav;
}
if (model.getFloor() != null && (model.getFloor() <= 0 || model.getFloor() > 59)) {
mav.setModel(BasicModel.buildBadRequestModel("floor取值非法,正确范围[1,60)"));
return mav;
}
if (model.getMaxFloor() != null && model.getFloor() != null && model.getFloor() > model.getMaxFloor()) {
mav.setModel(BasicModel.buildBadRequestModel("所在楼层不能大于总楼层"));
return mav;
}
HttpRequest request = new HttpRequest();
JSONObject json = null;
Map<String, String> params = new HashMap<String, String>();
params.put("client", model.getClient());
params.put("subjectMatter", model.getSubjectMatter());
params.put("city", model.getCity());
params.put("address", StringUtils.isNotBlank(model.getAddress()) ? model.getAddress() : null);
params.put("district", model.getDistrict());
params.put("name", StringUtils.isNotBlank(model.getName()) ? model.getName() : null);
params.put("houseNum", StringUtils.isNotBlank(model.getBuildingNumber()) ? model.getBuildingNumber() : null);
params.put("roomNum", StringUtils.isNotBlank(model.getUnitNumber()) ? model.getUnitNumber() : null);
params.put("size", String.valueOf(model.getSize()));
params.put("houseFloor", model.getFloor() != null ? String.valueOf(model.getFloor()) : null);
params.put("houseHeight", model.getMaxFloor() != null ? String.valueOf(model.getMaxFloor()) : null);
params.put("buildType", StringUtils.isNotBlank(model.getPropertyType()) ? model.getPropertyType() : "0");
params.put("direction", StringUtils.isNotBlank(model.getDirection()) ? model.getDirection() : "3");
params.put("fitment", StringUtils.isNotBlank(model.getDecoration()) ? model.getDecoration() : "2");
JSONObject jsonObject = JSONObject.fromObject(params);
Map<String, String> headers = new HashMap<String, String>();
headers.put("Authorization", "Bearer " + FangJiaTongUrl.FANG_JIA_TONG_TOKEN);
headers.put("Content-Type", "application/json");
try {
String ret = request.postWithBody(FangJiaTongUrl.FANG_JIA_TONG_URL_UAT + FangJiaTongUrl.ASSESS_PRICE_REPORT_APPLY, jsonObject, headers, "UTF-8");
json = JSONObject.fromObject(FangJiaTongUrl.replaceAll(ret));
} catch (Exception e) {
try {
String ret = request.postWithBody(FangJiaTongUrl.FANG_JIA_TONG_URL_UAT + FangJiaTongUrl.ASSESS_PRICE_REPORT_APPLY, jsonObject, headers, "UTF-8");
json = JSONObject.fromObject(FangJiaTongUrl.replaceAll(ret));
} catch (Exception ex) {
ex.printStackTrace();
}
}
if (json != null) {
if (json.getInt("code") == 200 && json.containsKey("data")) {
HashMap<String, String> result = new HashMap<String, String>();
result.put("caseId", String.valueOf(json.get("data")));
mav.setModel(BasicModel.buildOKModel().setResult(result));
} else {
mav.setModel(BasicModel.buildBadRequestModel("申请估值报告失败:" + json.getString("message")));
}
} else {
mav.setModel(BasicModel.buildBadRequestModel("申请估值报告失败:调用申请报告接口返回为空"));
}
return mav;
}
/**
* 获取估值报告
*
* @param caseId 估值报告ID
* @return
*/
public Object assessPriceReportFetchAction(@Param(name = "caseId") String caseId) {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
if (StringUtils.isBlank(caseId)) {
mav.setModel(BasicModel.buildBadRequestModel("参数非法:估值报告业务编号为必填项"));
return mav;
}
HttpRequest request = new HttpRequest();
JSONObject json = null;
Map<String, String> params = new HashMap<String, String>();
params.put("caseId", caseId);
JSONObject jsonObject = JSONObject.fromObject(params);
Map<String, String> headers = new HashMap<String, String>();
headers.put("Authorization", "Bearer " + FangJiaTongUrl.FANG_JIA_TONG_TOKEN);
headers.put("Content-Type", "application/json");
try {
String ret = request.postWithBody(FangJiaTongUrl.FANG_JIA_TONG_URL_UAT + FangJiaTongUrl.ASSESS_PRICE_REPORT_FETCH, jsonObject, headers, "UTF-8");
json = JSONObject.fromObject(FangJiaTongUrl.replaceAll(ret));
} catch (Exception e) {
try {
String ret = request.postWithBody(FangJiaTongUrl.FANG_JIA_TONG_URL_UAT + FangJiaTongUrl.ASSESS_PRICE_REPORT_FETCH, jsonObject, headers, "UTF-8");
json = JSONObject.fromObject(FangJiaTongUrl.replaceAll(ret));
} catch (Exception ex) {
ex.printStackTrace();
}
}
if (json != null) {
if (json.getInt("code") == 200 && json.containsKey("data")) {
JSONObject resultJson = json.getJSONObject("data");
mav.setModel(BasicModel.buildOKModel().setResult(resultJson));
} else {
mav.setModel(BasicModel.buildBadRequestModel("获取估值报告失败:" + json.getString("message")));
}
} else {
mav.setModel(BasicModel.buildBadRequestModel("申请报告失败:调用获取估值报告接口返回为空"));
}
return mav;
}
private Integer getFloor(String roomNumber) {
if (StringUtils.isNotBlank(roomNumber) && DataUtil.isNumber(roomNumber)) {
if (roomNumber.length() == 3) {
return Integer.valueOf(roomNumber.substring(0, 1));
} else if (roomNumber.length() == 4) {
return Integer.valueOf(roomNumber.substring(0, 2));
} else {
return null;
}
}
return null;
}
/**
* 通过EvaluationModel组装Evaluation实体
*
* @param model
* @param request
* @return
*/
private Evaluation buildEvaluation(PropertyEvaluationModel model, HttpServletRequest request) {
Evaluation evaluation = new Evaluation();
evaluation.setLevel(model.getLevel());
evaluation.setFitment(model.getFitment());
evaluation.setType(model.getType());
evaluation.setCity(model.getCity());
evaluation.setRegion(model.getRegion());
evaluation.setDistrictName(model.getName());
evaluation.setAddress(model.getAddress());
evaluation.setDirection(model.getDirection());
evaluation.setNoise(NOISE.get(model.getNoise()));
evaluation.setType(model.getType());
evaluation.setSight(SIGHT.get(model.getView()));
evaluation.setAtticArea(model.getAtticSize());
evaluation.setTerraceArea(model.getOuterSize());
evaluation.setBasementArea(model.getBasementSize());
evaluation.setGardenArea(model.getGardenSize());
String worryFacility = model.getNegativeFactor();
evaluation.setWorryFacility(NEGATIVE_FACTOR.get(worryFacility));
evaluation.setCreationDate(new Date());
evaluation.setModificationDate(new Date());
evaluation.setClientNo(model.getClientNo());
if (0 != model.getAge()) {
evaluation.setAge(model.getAge());
}
evaluation.setArea(model.getSize());
if (ArrayUtils.isNotEmpty(model.getBuildQuality())) {
evaluation.setBuildQuality(convertToString(model.getBuildQuality()));
}
if (StringUtils.isNotBlank(model.getBuildType())) {
evaluation.setBuildType(model.getBuildType());
}
if (StringUtils.isNotBlank(model.getCarport())) {
evaluation.setCarport(model.getCarport());
}
if (model.getDecorationAge() >= 1 && model.getDecorationAge() <= 5) {
evaluation.setFitmentAge(model.getDecorationAge());
}
evaluation.setCarportPrice(model.getCarportPrice());
evaluation.setCity(model.getCity());
evaluation.setDistrictName(model.getName());
if (model.getDecorationCost() != null && model.getDecorationCost() > 0) {
evaluation.setFitmentCharge(model.getDecorationCost());
}
if (0 != model.getHall()) {
evaluation.setHall(model.getHall());
}
if (null != model.getFloor()) {
evaluation.setFloor(model.getFloor());
}
if (model.getMaxFloor() != null && 0 != model.getMaxFloor()) {
evaluation.setHeight(model.getMaxFloor());
}
if (StringUtils.isNotBlank(model.getBuildingNumber())) {
evaluation.setHouseNum(model.getBuildingNumber());
}
if (StringUtils.isNotBlank(model.getHouseProperty())) {
evaluation.setHouseProperty(model.getHouseProperty());
}
if (StringUtils.isNotBlank(model.getInsidepos())) {
evaluation.setInsidepos(model.getInsidepos());
}
if (StringUtils.isNotBlank(model.getLightingVentilation())) {
evaluation.setLightingVentilation(model.getLightingVentilation());
}
if (StringUtils.isNotBlank(model.getOtherFactor())) {
evaluation.setOtherFactor(model.getOtherFactor());
}
evaluation.setRegion(model.getDistrict());
if (null != model.getRoom()) {
evaluation.setRoom(model.getRoom());
}
if (StringUtils.isNotBlank(model.getRoomNumber())) {
evaluation.setRoomNum(model.getRoomNumber());
}
if (StringUtils.isNotBlank(model.getSideset())) {
evaluation.setSideset(model.getSideset());
}
if (0 != model.getToilet()) {
evaluation.setToilet(model.getToilet());
}
if (model.getFacilities() != null) {
String facilities = model.getFacilities();
String[] fs = facilities.split(",");
for (int i = 0; i < fs.length; i++) {
fs[i] = FACILITIES.get(fs[i]);
}
evaluation.setFacilities(convertToString(fs));
}
if (StringUtils.isNotBlank(model.getFacade())) {
evaluation.setFacade(model.getFacade());
}
if (StringUtils.isNotBlank(model.getTraffice())) {
evaluation.setTraffice(model.getTraffice());
}
if (StringUtils.isNotBlank(model.getShape())) {
evaluation.setShape(model.getShape());
}
if (StringUtils.isNotBlank(model.getObstacle())) {
evaluation.setObstacle(model.getObstacle());
}
if (StringUtils.isNotBlank(model.getIndustry())) {
evaluation.setIndustry(model.getIndustry());
}
if (StringUtils.isNotBlank(model.getSupportDevice())) {
evaluation.setSupportDevice(model.getSupportDevice());
}
if (StringUtils.isNotBlank(model.getLeftPosition())) {
evaluation.setLeftPosition(model.getLeftPosition());
}
if (StringUtils.isNotBlank(model.getShopType())) {
evaluation.setShopType(model.getShopType());
}
if (StringUtils.isNotBlank(model.getShopUnique())) {
evaluation.setShopUnique(model.getShopUnique());
}
if (StringUtils.isNotBlank(model.getLocation())) {
evaluation.setLocation(model.getLocation());
}
if (model.isNewDistrict()) {
evaluation.setNewDistrict(model.isNewDistrict());
}
String floorDesc = "中";
if (evaluation.getFloor() != null && evaluation.getHeight() != null) {
Double floorInfo = evaluation.getFloor() * 1.0 / evaluation.getHeight();
if (floorInfo <= 0.33) {
floorDesc = "低";
} else if (floorInfo > 0.66) {
floorDesc = "高";
}
evaluation.setFloorDesc(floorDesc);
}
String ip = AnalyticsUtil.getIpAddress(ActionContext.getContext().getRequest());
evaluation.setIp(ip);
AuthenticationBean authenticationBean = (AuthenticationBean) request
.getAttribute(AuthenticationBean.ATTR_AUTHENTICATION_BEAN);
if (authenticationBean != null && authenticationBean.apiUser != null) {
evaluation.setFromUserId(String.valueOf(authenticationBean.apiUser.getId()));
}
evaluation.setSource("api_shop_evaluateRent");
return evaluation;
}
private static final Map<String, String> FACILITIES = new HashMap<String, String>() {
{
put("0", "健身房");
put("1", "游泳池");
put("2", "会所");
put("3", "网球场");
put("4", "球场");
put("5", "棋牌室");
put("6", "停车场");
put("7", "图书馆");
put("8", "活动中心");
}
};
public void setOpenApiLogService(OpenApiLogService openApiLogService) {
this.openApiLogService = openApiLogService;
}
public void setFieldFilterService(FieldFilterService fieldFilterService) {
this.fieldFilterService = fieldFilterService;
}
public void setAiShopService(AiShopService aiShopService) {
this.aiShopService = aiShopService;
}
}
+12
View File
@@ -543,6 +543,18 @@
<version>1.18.2</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-text</artifactId>
<version>1.9</version>
</dependency>
<dependency>
<groupId>com.hankcs.hanlp.restful</groupId>
<artifactId>hanlp-restful</artifactId>
<version>0.0.12</version>
</dependency>
<!-- <dependency>-->
<!-- <groupId>com.fangjia</groupId>-->
<!-- <artifactId>fangjia-open</artifactId>-->
@@ -21,7 +21,7 @@ import java.sql.DriverManager
@BatchOwner(Watcher.WangWenYan)
class ClickHouseDistrictStatMarkupCalcBatch extends BaseBatch {
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 27017)
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo", 27017)
static void main(String[] args) {
runBatch(new ClickHouseDistrictStatMarkupCalcBatch(), args)
}
@@ -18,7 +18,7 @@ import com.mongodb.DBCollection
@BatchOwner(Watcher.WangWenYan)
class DistrictStatMarkupCalcBatch extends BaseBatch {
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 27017)
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com", 27017)
Date oneYearAgo = DateUtil.add(new Date(), Calendar.MONTH, -1)
@@ -8,8 +8,10 @@ import com.mongodb.DBCollection
/**
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2025年01月深圳二手房成交的.xlsx", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
//没有成交时间,随机生成
@@ -18,7 +18,7 @@ import java.util.concurrent.atomic.AtomicLong
*/
class AddThirdPriceSample {
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
public static final DBCollection NO_PRICE_DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "deal_price_all", "mongo.db.fangjia.com")
@@ -43,7 +43,7 @@ class BigdataDealBatch extends BaseBatch{
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
@@ -45,7 +45,7 @@ class DistrictDelete {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
static main(args) {
def task = new DistrictDelete()
@@ -27,7 +27,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07",
@@ -36,7 +36,7 @@ class TradeStatLog {
def pssqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
@@ -18,10 +18,10 @@ import com.mongodb.QueryOperators
import groovy.sql.Sql
DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
DBCollection CITY_STAT = MongoUtils.getCollectionByName("fangjia_stat", "city_stat", "192.168.0.43")
DBCollection CITY_INFO = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", "192.168.0.43")
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
DBCollection CITY_STAT = MongoUtils.getCollectionByName("fangjia_stat", "city_stat", "mongo.db.fangjia.com")
DBCollection CITY_INFO = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", "mongo.db.fangjia.com")
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
@@ -12,7 +12,7 @@ class MatchDealPriceRegion {
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
static main(args) {
@@ -21,7 +21,7 @@ import org.apache.lucene.queryparser.classic.QueryParser
*/
class EvaluationBatch {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
@@ -18,7 +18,7 @@ class FumingTest {
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver")
@@ -38,7 +38,7 @@ class HailanTest {
}
static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
@@ -19,7 +19,7 @@ import java.util.concurrent.atomic.AtomicLong
class PocStat {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", lineUrl)
public static final DBCollection LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia_base", "loudong", lineUrl)
@@ -15,7 +15,7 @@ class RuiFenPocTest {
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver")
@@ -99,7 +99,7 @@ class CommonDatautils {
}
public static final DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009")
public static final DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
public static final DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com", 28017)
public static final DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com")
public static final DBCollection STATS_INDEX_COLL = MongoUtils.getCollectionByName("fangjia_stat", "stats_index", "mongo.db.fangjia.com")
@@ -23,7 +23,7 @@ import org.apache.commons.lang.StringUtils
class FewerHouseDistrictCrawlCommandBatch extends BaseBatch {
private static
final DBCollection ARTEMIS_COLL = MongoUtils.getCollectionByName("fangjia_log", "artemis_message", "mongo.db.fangjia.com")
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43")
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static void main(String[] args) {
runBatch(new FewerHouseDistrictCrawlCommandBatch(), args)
@@ -15,7 +15,7 @@ import org.bson.types.ObjectId
//new FewerHouseDistrictCrawlCommandBatch().process(null);
//return;
//
//DBCollection col_43_artemis_message = MongoUtils.getCollectionByName("fangjia_log", "artemis_message", "192.168.0.43", 27017);
//DBCollection col_43_artemis_message = MongoUtils.getCollectionByName("fangjia_log", "artemis_message", "mongo.db.fangjia.com", 27017);
//DBCursor cursor_artemis_message = col_43_artemis_message.find(new BasicDBObject("status", 0));
//int count = 0;
//while(cursor_artemis_message.hasNext()){
@@ -21,8 +21,8 @@ import java.util.concurrent.TimeUnit
@BatchOwner(Watcher.GaoJuan)
class DistrictTradeHouseStatBatch extends BaseBatch{
public static DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
public static DBCollection DISTRICT_TRADE_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_trade_stat", "192.168.0.43")
public static DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
public static DBCollection DISTRICT_TRADE_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_trade_stat", "mongo.db.fangjia.com")
static main(args) {
runBatch(new DistrictTradeHouseStatBatch(), args)
@@ -23,7 +23,7 @@ import org.elasticsearch.common.xcontent.XContentType
*/
@BatchOwner(Watcher.GaoJuan)
class SynchDealPriceToEs extends BaseBatch{
static DBCollection dealPriceConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017)
static DBCollection dealPriceConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com",28017)
static main(args) {
runBatch(new SynchDealPriceToEs(), args)
@@ -20,7 +20,7 @@ class AbcEvaluationRowsBatch extends BaseBatch{
static final def fjtSqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/goojia_fangjia?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver");
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", dbUrl)
static main(args) {
@@ -29,7 +29,7 @@ class DistrictInfoExportBatch extends BaseBatch{
def charset = "GBK"
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection FZ_ADDR_COLL = MongoUtils.getCollectionByName2("business", "fz_district_address", lineUrl)
public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName2("business", "fz_district", lineUrl)
@@ -23,7 +23,7 @@ class FzDataResultStat {
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName2("business", "fz_district", lineUrl)
@@ -16,7 +16,7 @@ import net.sf.json.JSONObject
import org.apache.commons.io.FileUtils
import org.bson.types.ObjectId
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
import com.fangjia.core.util.HttpRequest
@@ -169,7 +169,7 @@ DingDingMessageUtil.sendTextMessage(access_token, s)
//
//
//def count(name, sDate){
// DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
// DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
// def q = new BasicDBObject()
// q.put("city","¸£ÖÝ")
// q.put("s_date", sDate)
@@ -21,7 +21,7 @@ class ThirdPriceExportBatch extends BaseBatch{
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
@@ -16,7 +16,7 @@ import java.util.concurrent.CountDownLatch
class ThirdTest {
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
static main(args) {
@@ -25,7 +25,7 @@ class TradeExportBatch extends BaseBatch{
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
static def districtMap = [:]
@@ -30,7 +30,7 @@ import java.util.concurrent.atomic.AtomicInteger
class CzBankStat {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("business", "average_statistics", "192.168.0.43")
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("business", "average_statistics", "mongo.db.fangjia.com")
def static final DBCollection CZLOG = MongoUtils.getCollectionByName("fangjia_log", "cz_log2", "192.168.0.136")
static main(args) {
def task = new CzBankStat();
@@ -16,7 +16,7 @@ import org.apache.commons.lang.StringUtils
import java.util.regex.Pattern
class DealPriceProcess {
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
def total = 0
static main(args) {
@@ -17,7 +17,7 @@ import java.util.concurrent.Executors
class DistrictDetailBatch {
public static DBCollection DISTRICT_DETAIL_COLL = MongoUtils.getCollectionByName("fangjia_youfang", "district_detail", "192.168.0.43")
public static DBCollection DISTRICT_DETAIL_COLL = MongoUtils.getCollectionByName("fangjia_youfang", "district_detail", "mongo.db.fangjia.com")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
@@ -40,7 +40,7 @@ class EsHouseStat {
dataSource.setProperties(connectionProperties)
}
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
public static void main(String[] args){
@@ -16,8 +16,8 @@ import java.sql.ResultSet
class FZDataExport {
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
@@ -8,7 +8,7 @@ import com.mongodb.DBCollection
class FZHouseEvaluation {
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", dbUrl)
def regionMap = [:]
@@ -8,7 +8,7 @@ import com.mongodb.DBCollection
class FjwUserStat {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection USER_COLL = MongoUtils.getCollectionByName2("fangjia_coresite", "user", lineUrl)
@@ -18,7 +18,7 @@ import org.bson.Document;
import org.bson.conversions.Bson;
class HouseLouDongMatch {
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", dbUrl)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
@@ -22,7 +22,7 @@ class LandDetailCrawl {
DBCollection LAND_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "land_deal_price1", "192.168.0.235", 27018)
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
public static final DBCollection FJT_LAND_COLL = MongoUtils.getCollectionByName2("business", "land_distinct", lineUrl)
public static final DBCollection OFF_FJT_LAND_COLL = MongoUtils.getCollectionByName2("business", "land_distinct", offlineUrl)
@@ -14,10 +14,10 @@ import org.apache.lucene.queryparser.classic.QueryParser
class LogStat {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
static main(args) {
def task = new LogStat()
@@ -21,8 +21,8 @@ import java.util.concurrent.Executors
@BatchOwner(Watcher.GaoJuan)
class NoHouseDistrictStatBatch extends BaseBatch{
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
// public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
// public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", dbUrl)
public static final DBCollection AVERAGER_CONFIG_COLL = MongoUtils.getCollectionByName2("fangjia", "averager_config", dbUrl)
@@ -13,7 +13,7 @@ class PingAnAddressLevalTest {
String token = "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB";
DBCollection DIST_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_grade", "192.168.0.43")
DBCollection DIST_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_grade", "mongo.db.fangjia.com")
static main(args) {
def task = new PingAnAddressLevalTest()
task.run();
@@ -329,7 +329,7 @@ def findDistrictPrice(def city, def region){
q.put("region", region);
}
//districtStatConn.addOption(Bytes.QUERYOPTION_SLAVEOK);
final def districtStatConn = Mongo3Utils.getDbCollection("mongodb://developer:G00IDCDEV12385@192.168.0.43:28017","fangjia_stat", "district_stat");
final def districtStatConn = Mongo3Utils.getDbCollection("mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017","fangjia_stat", "district_stat");
def count = districtStatConn.count(q)
return count;
}
@@ -343,7 +343,7 @@ def findDistrictPrice(def city, def region){
q.put("region", region);
}
//districtStatConn.addOption(Bytes.QUERYOPTION_SLAVEOK);
final def districtStatConn = Mongo3Utils.getDbCollection("mongodb://developer:G00IDCDEV12385@192.168.0.43:28017","fangjia_stat", "district_stat");
final def districtStatConn = Mongo3Utils.getDbCollection("mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017","fangjia_stat", "district_stat");
def count = districtStatConn.count(q)
return count;
}
@@ -6,7 +6,7 @@ import com.mongodb.DBCollection
class RegionBlockSetting {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection REGION_BLOCK_COLL = MongoUtils.getCollectionByName2("fangjia", "region_block", lineUrl)
static main(args) {
@@ -17,8 +17,8 @@ import com.mongodb.DBObject
class SynchDistrictStatBatch extends BaseBatch{
//public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 28017)
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
//public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com", 28017)
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", lineUrl)
@@ -15,7 +15,7 @@ import java.util.concurrent.atomic.AtomicLong
class UpdateThirdPrice {
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
def static final DBCollection AVERAGE_STATISTICS_OFFLINE = CrawalDBUtil.getAverage_statistics_col();
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
@@ -14,7 +14,7 @@ import groovy.sql.Sql
class ApiUserCountAnalyBatch extends BaseBatch{
static final dbFangjia = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia?autoReconnect=true&characterEncoding=utf-8&autoReconnect=true", "sa", "G00jia", "com.mysql.jdbc.Driver");
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection AUTHORITY_COLL = MongoUtils.getCollectionByName2("fangjia", "authority.api_user_authority", lineUrl)
DBCollection VISIT_COLL = MongoUtils.getCollectionByName2("business", "api_visit_count", lineUrl)
@@ -10,7 +10,7 @@ import com.mongodb.DBCollection
class CcbCityPriceStat {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
static main(args) {
@@ -20,7 +20,7 @@ import com.mongodb.DBCollection
*/
class PocTest2023 {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
public static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
@@ -15,7 +15,7 @@ import org.apache.commons.lang.math.RandomUtils
class DistrictSelectHouse {
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
static main(args) {
def task = new DistrictSelectHouse();
@@ -9,8 +9,8 @@ import com.mongodb.QueryOperators
class TestPrice {
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
static main(args) {
@@ -17,7 +17,7 @@ import org.elasticsearch.index.query.QueryBuilders
class CityStatExport {
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
public static final def dbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", dbUrl)
@@ -16,7 +16,7 @@ import java.util.concurrent.atomic.AtomicLong
class CityStat {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", lineUrl)
public static final DBCollection BAD_CASE_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "bad_case", lineUrl)
@@ -21,13 +21,13 @@ import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
DBCollection region_stat_coll = MongoUtils.getCollectionByName("fangjia", "region_stat", "192.168.0.43")
DBCollection region_stat_coll = MongoUtils.getCollectionByName("fangjia", "region_stat", "mongo.db.fangjia.com")
DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020)
//def citys = ["ÎÚ³ľÆë","²ý¼ª","°¢¿ËËÕ","ÒÁÀç","¿¦Ê²"]
@@ -11,7 +11,7 @@ import com.mongodb.QueryOperators
class BestDistrictPriceExport {
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
static main(args) {
def task = new BestDistrictPriceExport();
@@ -2,12 +2,18 @@ package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.BaseBatch
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CopyOnWriteArraySet
@@ -16,30 +22,47 @@ import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class AutoAddOfficeBatch extends BaseBatch{
class AutoAddOfficeBatch extends BaseBatch {
def fullCityMap = CommonDatautils.queryFullNameByCity()
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
def fullCityMap = CommonDatautils.queryFullNameByCity()
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206", 27020)
//235为房价网内部计算均价的库
def pgsqlInstance235 = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
static main(args) {
// runBatch(new AutoAddOfficeBatch(), args)
def task = new AutoAddOfficeBatch()
task.repeatOffice()
// task.repeatOffice()
task.repeatDist()
}
@Override
void process(Object args) {
// def pattern = /[A-Z0-9]座|[0-9a-z]栋|[A-Z]栋|·|\(建设中\)|[0-9]号楼|\(装修中\)|[A-Z]楼|[0-9]幢/
// def newName = "开放广场".replaceAll(pattern, "")
// println newName
// findAddOfficeDatas() //查找需要新增的写字楼数据
// checkInBaiduMapOffice()//查寻在百度地图上有的写字楼数据
// addOffice()//新增写字楼
}
def findAddOfficeDatas() {
def citys = CommonDatautils.findAllCity()
def allAddOffices = []
allAddOffices << ["地图ID","省","市","区","写字楼名称","地址","经度","纬度","匹配城市","匹配区域","匹配名称","匹配方式"]
for(def c : citys){
allAddOffices << ["地图ID", "省", "市", "区", "写字楼名称", "地址", "经度", "纬度", "匹配城市", "匹配区域", "匹配名称", "匹配方式"]
for (def c : citys) {
def cityObj = fullCityMap.get(c)
if(cityObj){
if (cityObj) {
def officeMaps = CommonDatautils.findOfficeByMap(cityObj.city, "商务写字楼")
officeMaps = filterOffice(officeMaps)
if(officeMaps){
if (officeMaps) {
def offices = matchOffice(officeMaps)
allAddOffices.addAll(offices)
@@ -49,59 +72,72 @@ class AutoAddOfficeBatch extends BaseBatch{
ExcelUtils.write("D:\\数据\\高德写字楼.xlsx", allAddOffices, "数据")
}
def filterOffice(officeMaps){
def checkInBaiduMapOffice() {
ExcelUtils.multiThreadingModify("D:\\数据\\高德写字楼1.xlsx", { d ->
if (!d['L']) {
def obj = findBaiduDatas(d['C'], d['D'], d['E'])
if (obj) {
d['L'] = "百度存在"
d['M'] = obj.std_tag.toString()
}
}
}, 0)
}
def filterOffice(officeMaps) {
def offices = []
def keys = []
for(def o : officeMaps){
if(!o){
for (def o : officeMaps) {
if (!o) {
continue
}
if(o.name.startsWith("写字楼") || o.name.startsWith("综合楼") || o.name.contains("销售") || o.name.startsWith("办公楼") || o.name.startsWith("公务楼") || o.name.startsWith("行政楼") || o.name.startsWith("商务办公楼") || o.name.startsWith("综合办公楼")){
println "[ 过滤 ] "+o.city+" "+o.name
if (o.name.startsWith("写字楼") || o.name.startsWith("综合楼") || o.name.contains("销售") || o.name.startsWith("办公楼") || o.name.startsWith("公务楼") || o.name.startsWith("行政楼") || o.name.startsWith("商务办公楼") || o.name.startsWith("综合办公楼")) {
println "[ 过滤 ] " + o.city + " " + o.name
continue
}
if(o.name.contains("路写字楼")){
println "[ 过滤 ] "+o.city+" "+o.name
if (o.name.contains("路写字楼")) {
println "[ 过滤 ] " + o.city + " " + o.name
continue
}
def pattern = /[A-Z0-9]座|[0-9a-z]栋|[A-Z]栋|·|\(建设中\)|[0-9]号楼|\(装修中\)|[A-Z]楼|[0-9]幢/
def newName = o.name.replaceAll(pattern, "")
o.name = newName
def keyName = o.city +"#"+o.district+"#"+o.name
if(!keys.contains(keyName)){
def keyName = o.city + "#" + o.district + "#" + o.name
if (!keys.contains(keyName)) {
offices << o
keys << keyName
}
}
println "原:"+officeMaps.size() +" 新:"+offices.size()
println "原:" + officeMaps.size() + " 新:" + offices.size()
return offices
}
def matchOffice(offices){
def matchOffice(offices) {
def addCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(offices.size())
def total = offices.size()
def rets = new CopyOnWriteArrayList()
for(def i=0;i<offices.size();i++){
for (def i = 0; i < offices.size(); i++) {
final def nmd = offices[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
try {
addCount.addAndGet(1)
def office = matchDB(nmd)
if(!office){
if (!office) {
office = matchApi(nmd)
}
if(office){
if (office) {
rets << [nmd.map_id, nmd.province, nmd.city, nmd.district, nmd.name, nmd.address, nmd.lng, nmd.lat, office.city, office.region, office.name, office.match_way]
println addCount.get()+"/"+total +" [success] "+ office.get("region")+" "+office.get("name")+" "+office.match_way
}else{
rets << [nmd.map_id, nmd.province, nmd.city, nmd.district, nmd.name, nmd.address,nmd.lng, nmd.lat,"","","",""]
println addCount.get()+"/"+total +" [fail] "+nmd.city+" "+nmd.name
println addCount.get() + "/" + total + " [success] " + office.get("region") + " " + office.get("name") + " " + office.match_way
} else {
rets << [nmd.map_id, nmd.province, nmd.city, nmd.district, nmd.name, nmd.address, nmd.lng, nmd.lat, "", "", "", ""]
println addCount.get() + "/" + total + " [fail] " + nmd.city + " " + nmd.name
}
} catch (Exception e) {
e.printStackTrace()
@@ -115,28 +151,28 @@ class AutoAddOfficeBatch extends BaseBatch{
return rets
}
def matchDB(nmd){
def matchDB(nmd) {
def simpleCity = RegionData.formatCity(nmd.city)
def simpleDistrict = RegionData.formatRegion(simpleCity, nmd.district)
if(simpleCity && simpleDistrict){
if (simpleCity && simpleDistrict) {
def office = null
def rets = CommonDatautils.findDistrictList(simpleCity, simpleDistrict, nmd.name, "office")
if(rets){
if(rets.size()>=1){
if (rets) {
if (rets.size() >= 1) {
office = rets[0]
}
}else{
} else {
rets = CommonDatautils.findDistrictByAlias(simpleCity, simpleDistrict, nmd.name, "office")
if(rets && rets.size()>=1){
if (rets && rets.size() >= 1) {
office = rets[0]
}else{
} else {
rets = CommonDatautils.findDistrictByAddress(simpleCity, simpleDistrict, nmd.name, "district")
if(rets && rets.size()>=1){
if (rets && rets.size() >= 1) {
office = rets[0]
}
}
}
if(office){
if (office) {
office.match_way = "数据库"
}
return office
@@ -144,65 +180,152 @@ class AutoAddOfficeBatch extends BaseBatch{
return null
}
def matchApi(nmd){
def matchApi(nmd) {
def address = nmd.city
if(nmd.district){
if (nmd.district) {
address = address + nmd.district
}
address = address + nmd.name
def jsonObject = FangjiaClientUtils.newMatch(nmd.city, address, "office", false)
if(!jsonObject){
if(DataUtil.isPreciseAddress(nmd.address)){
if (!jsonObject) {
if (DataUtil.isPreciseAddress(nmd.address)) {
address = nmd.city
if(nmd.district){
if (nmd.district) {
address = address + nmd.district
}
address = address + nmd.address
jsonObject = FangjiaClientUtils.newMatch(nmd.city, address, "office", false)
}
}
if(jsonObject) {
def udata = [:]
udata.put("_id",jsonObject.get("id"))
udata.put("city",jsonObject.get("city"))
udata.put("region",jsonObject.get("district"))
udata.put("name",jsonObject.get("name"))
udata.put("match_type",jsonObject.get("propertyType"))
udata.put("match_credit",jsonObject.get("credit"))
if (jsonObject) {
def udata = [:]
udata.put("_id", jsonObject.get("id"))
udata.put("city", jsonObject.get("city"))
udata.put("region", jsonObject.get("district"))
udata.put("name", jsonObject.get("name"))
udata.put("match_type", jsonObject.get("propertyType"))
udata.put("match_credit", jsonObject.get("credit"))
udata.put("match_way", "接口")
return udata
}
return null
}
def repeatOffice(){
def repeatOffice() {
def citys = CommonDatautils.findAllCity()
def allrets = []
allrets << ["ID","城市","区域","名称","地址","可疑ID","可疑城市","可疑区域","可疑名称","可疑地址","相似度","距离"]
for(def c : citys){
def offices = CommonDatautils.findAllDistricts(c, "office", ["city","region","name","address", "lat", "lng"])
allrets << ["ID", "城市", "区域", "名称", "地址", "可疑ID", "可疑城市", "可疑区域", "可疑名称", "可疑地址", "相似度", "距离"]
for (def c : citys) {
def offices = CommonDatautils.findAllDistricts(c, "office", ["city", "region", "name", "address", "lat", "lng"])
def rets = checkRepeatOfficeBatch(offices)
println c +" "+offices.size()
println c + " " + offices.size()
allrets.addAll(rets)
}
ExcelUtils.write("D:\\数据\\写字楼\\重复可疑写字楼1.xlsx", allrets, "数据")
}
def checkRepeatOfficeBatch(offices){
def repeatDist(){
def citys = CommonDatautils.findAllCity()
def allrets = []
allrets << [ "城市", "区域", "名称", "别名","计算方式","地址","名称相似度","地址相似度","总相似度"]
def index = 0
for (def c : citys) {
index++
if(index>4){
continue
}
def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "address", "alias"])
def rets = checkRepeatDistrictBatch(c, districts)
println c + " " + districts.size()
allrets.addAll(rets)
println "${index}"+ c +" 运A行完成 "+rets.size()
}
ExcelUtils.write("/home/goojia/gj/重复可疑小区1.xlsx", allrets, "数据")
}
def checkRepeatDistrictBatch(city,districts) {
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def cityId = cityCodeDataMap.get(city)
def fjwPriceMap = findNewFjwAvgPriceList(cityId, 202502)
def addCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
def total = districts.size()
def rets = new CopyOnWriteArrayList()
def sets = new CopyOnWriteArraySet()
for (def i = 0; i < total; i++) {
final def o1 = districts[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
try {
addCount.addAndGet(1)
// if(!o1.name.contains("四公司")){
// continue
// }
if(!sets.contains(o1._id.toString())){
def similarRets = findListSimilar(o1, districts)
if (similarRets) {
def newPriceMap1 = fjwPriceMap.get(o1._id.toString())
def new_price1 = null
if(newPriceMap1){
new_price1 = newPriceMap1.calc_type
}
def ailas_str1 = ""
if(o1.alias){
ailas_str1 = o1.alias.join(",")
}
def rows = []
for(def similar : similarRets){
if (!sets.contains(similar._id.toString())) {
sets.add(similar._id.toString())
def ailas_str = ""
if(similar.alias){
ailas_str = similar.alias.join(",")
}
def newPriceMap = fjwPriceMap.get(o1._id.toString())
def new_price = null
if(newPriceMap){
new_price = newPriceMap.calc_type
}
rows << [ similar.city, similar.region, similar.name, ailas_str,new_price, similar.address, similar.nameScore, similar.addrScore, similar.score]
// rets << [ similar.city, similar.region, similar.name, ailas_str,new_price, similar.address, similar.nameScore, similar.addrScore, similar.score]
}
}
if(rows){
sets.add(o1._id.toString())
rets << [ o1.city, o1.region, o1.name, ailas_str1,new_price1, o1.address, "主", "主", "主"]
rets.addAll(rows)
}
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
// }
// });
}
// latch.await()
return rets
}
def checkRepeatOfficeBatch(offices) {
def addCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(offices.size())
def total = offices.size()
def rets = new CopyOnWriteArrayList()
def sets = new CopyOnWriteArraySet()
for(def i=0;i<total;i++){
for (def i = 0; i < total; i++) {
final def o1 = offices[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
try {
addCount.addAndGet(1)
def maxObj = findOneSimilarOffice(o1, offices)
if(maxObj){
if(!sets.contains(o1._id.toString())){
if (maxObj) {
if (!sets.contains(o1._id.toString())) {
sets.add(maxObj._id.toString())
rets << [o1._id.toString(), o1.city, o1.region, o1.name, o1.address, maxObj._id.toString(), maxObj.city, maxObj.region, maxObj.name, maxObj.address, maxObj.score, maxObj.distance]
}
@@ -219,28 +342,215 @@ class AutoAddOfficeBatch extends BaseBatch{
return rets
}
def findOneSimilarOffice(o1, offices){
def findListSimilar(o1, datas){
def rets = new CopyOnWriteArrayList()
def ids = []
ids << o1._id.toString()
for (def o2 : datas) {
if (o1._id.toString() != o2._id.toString() && o1.region==o2.region) {
// if(!o2.name.contains("四公司")){
// continue
// }
def name1 = o1.name//.replace("小区","").replace("别墅","")
def name2 = o2.name//.replace("小区","").replace("别墅","")
def nameScore = CosineSimilarAlgorithm.levenshteinSimilarity(name1, name2);
def addrScore = null
if(o1.address && o2.address){
addrScore = CosineSimilarAlgorithm.levenshteinSimilarity(o1.address, o2.address);
}
def str = null
if(o1.name.length() > o2.name.length()){
str = o1.name.replace(o2.name, "")
}else{
str = o2.name.replace(o1.name, "")
}
if (nameScore > 0.6 && !["一期","二期","三期","四期","五期","六期","七期","八期","九期","十期","东区","西区","南区","北区","A区","B区","C区","D区"].contains(str)) {
o2.nameScore = nameScore
o2.addrScore = addrScore
o2.score = nameScore
rets << o2
ids << o2._id.toString()
def maxObj = findMaxSimilar(ids, o2, datas)
while(maxObj){
maxObj.nameScore = CosineSimilarAlgorithm.levenshteinSimilarity(maxObj.name, o1.name);
if(o1.address && maxObj.address){
maxObj.addrScore = CosineSimilarAlgorithm.levenshteinSimilarity(o1.address, maxObj.address);
}
maxObj.score = maxObj.nameScore
rets << maxObj
ids << maxObj._id.toString()
maxObj = findMaxSimilar(ids, maxObj, datas)
}
}
}
}
rets.sort{a,b->
def a1 = a.score
def b1 = b.score
return b1.compareTo(a1)
}
if (rets.size()>2) {
return rets[0..2]
}
return rets
}
def findMaxSimilar(ids, o1, datas){
def maxObj = null
def maxScore = -1000
for (def o2 : datas) {
if (o1._id.toString() != o2._id.toString() && o1.region==o2.region && !ids.contains(o2._id.toString())) {
def name1 = o1.name//.replace("小区","").replace("别墅","")
def name2 = o2.name//.replace("小区","").replace("别墅","")
def nameScore = CosineSimilarAlgorithm.levenshteinSimilarity(name1, name2);
def addrScore = null
if(o1.address && o2.address){
addrScore = CosineSimilarAlgorithm.levenshteinSimilarity(o1.address, o2.address);
}
def str = null
if(o1.name.length() > o2.name.length()){
str = o1.name.replace(o2.name, "")
}else{
str = o2.name.replace(o1.name, "")
}
if (nameScore > 0.6 && nameScore>maxScore && !["一期","二期","三期","四期","五期","六期","七期","八期","九期","十期","东区","西区","南区","北区","A区","B区","C区","D区"].contains(str)) {
maxObj = o2
o2.nameScore = nameScore
o2.addrScore = addrScore
o2.score = nameScore
}
}
}
return maxObj
}
def findOneSimilarOffice(o1, offices) {
def maxScore = 0
def maxObj = null
for(def o2 : offices){
if(o1._id.toString()!=o2._id.toString()){
for (def o2 : offices) {
if (o1._id.toString() != o2._id.toString()) {
double nameScore = CosineSimilarAlgorithm.getSimilarity(o1.name, o2.name);
double addrScore = CosineSimilarAlgorithm.getSimilarity(o1.address,o2.address);
if(nameScore+addrScore>maxScore){
maxScore = nameScore+addrScore
double addrScore = CosineSimilarAlgorithm.getSimilarity(o1.address, o2.address);
if (nameScore + addrScore > maxScore) {
maxScore = nameScore + addrScore
maxObj = o2
}
}
}
def distance = -1000
if(o1.lat && maxObj && maxObj.lat){
distance = GeoHelper.GetDistance(o1.lat/10000000000.0, o1.lng/10000000000.0, maxObj.lat/10000000000.0, maxObj.lng/10000000000.0)
if (o1.lat && maxObj && maxObj.lat) {
distance = GeoHelper.GetDistance(o1.lat / 10000000000.0, o1.lng / 10000000000.0, maxObj.lat / 10000000000.0, maxObj.lng / 10000000000.0)
}
if(maxObj){
if (maxObj) {
maxObj.distance = distance
maxObj.score = maxScore
return maxObj
}
return null
}
def findBaiduDatas(city, district, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("district", district)
q.put("name", name)
// q.put('std_tag', '房地产;住宅区')
def obj = BAIDU_POI_COLL.findOne(q)
return obj;
}
def addOffice(){
ExcelUtils.modify("D:\\数据\\高德待入库写字楼.xlsx",{d->
if(d['L']!='人工删除'){
def city = RegionData.formatCity(d['C'])
def region = RegionData.formatRegion(city, d['D'])
if(city && region){
def locationstr = d['G'].toString()+","+d['H'].toString()
def bdpoint = FangjiaClientUtils.convert(locationstr, 1)
def udata = new BasicDBObject()
if(bdpoint){
udata.put("lng2", (bdpoint[0]*10000000000).longValue())
udata.put("lat2", (bdpoint[1]*10000000000).longValue())
}
udata.put("city", city)
udata.put("region", region)
udata.put("name", d['E'])
udata.put("address", d['I'])
udata.put("cat", "office")
udata.put("status", 0)
udata.put("c_date",new Date())
udata.put("m_date",new Date())
udata.put("lng", (d['G'].toString().toDouble()*10000000000).longValue())
udata.put("lat", (d['H'].toString().toDouble()*10000000000).longValue())
def estate_type = null
if(d['M'] && d['M'].contains("园区")){
estate_type = "创意园区"
}
if(!estate_type && d['E'].contains("创意")){
estate_type = "创意园区"
}
if(estate_type){
udata.put("estate_type2", estate_type)
}
def markMap = [:]
markMap.put("data_source", "机器入库")
udata.put("mark_map", markMap)
DbUtils.seaweed.save(udata)
println d['E'] +" "+udata._id.toString()
}
}
})
}
def findNewFjwAvgPrice(def commid, def cityid){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from price_stat.avm_community_base_price where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
def map = [:]
pgsqlInstance235.eachRow(sql, { r ->
map.avg_price = r.gujia_bds_clc_avprc
map.type = r.gujia_bds_clmtd
map.month = r.gujiabds_clc_yrmo_val
map.credit_value = r.gujia_bds_cfdc_dgr
if(r.gujia_bds_clmtd == 1){
map.calc_type = "人工估值"
}else if(r.gujia_bds_clmtd == 2){
map.calc_type = "样本计算"
}else if(r.gujia_bds_clmtd == 3){
map.calc_type = "神经网络"
}
})
return map
}
def findNewFjwAvgPriceList(def cityid, date){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr,gujia_bds_id from price_stat.avm_community_base_price where urbn_id='${cityid}' and gujiabds_clc_yrmo_val=${date}"
def retMap = [:]
pgsqlInstance235.eachRow(sql, { r ->
def map = [:]
map.avg_price = r.gujia_bds_clc_avprc
map.type = r.gujia_bds_clmtd
map.month = r.gujiabds_clc_yrmo_val
map.credit_value = r.gujia_bds_cfdc_dgr
if(r.gujia_bds_clmtd == 1){
map.calc_type = "人工估值"
}else if(r.gujia_bds_clmtd == 2){
map.calc_type = "样本计算"
}else if(r.gujia_bds_clmtd == 3){
map.calc_type = "神经网络"
}
retMap.put(r.gujia_bds_id, map)
})
return retMap
}
}
@@ -5,10 +5,8 @@ import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import java.util.regex.Pattern
/**
* Ãû¼¼¯
*/
@@ -34,6 +32,7 @@ class MinglujiData {
def prov = "½­ËÕÊ¡"
def city = "ÎÞÎý"
exportDatas(city)
// exportDatas(city)
// matchDev(prov, city)
// matchCommunity(city)
// fiveLevelSplit(city)
@@ -4,9 +4,12 @@ import com.fangjia.core.RegionData
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import org.apache.commons.io.FileUtils
import org.bson.types.ObjectId
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.update.UpdateRequest
import org.elasticsearch.client.RequestOptions
@@ -33,10 +36,14 @@ class RegionStandardBatch extends BaseBatch{
void process(Object args) {
seaweedRegionStandard()
// poiRegionStandard()
// seaweedRegionStandard()
// poiRegionStandard()
//通过乡镇街道修改区域 如:东莞市
// updateRegionByTown()
}
def seaweedRegionStandard(){
seaweedRegionStandardByCat("district")
seaweedRegionStandardByCat("office")
}
def poiRegionStandard(){
@@ -93,7 +100,7 @@ class RegionStandardBatch extends BaseBatch{
def seaweedRegionStandardByCat(cat){
def citys = CommonDatautils.findAllCity()
def citys = ["伊春"]//CommonDatautils.findAllCity()
for(def city : citys){
def rMap = regionMap.get(city)
@@ -148,8 +155,12 @@ class RegionStandardBatch extends BaseBatch{
def streetMap = CommonDatautils.queryAllStreetNameByCity(fullCityName)
if(streetMap){
set = streetMap.keySet()
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name"])
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name","lng","lat"])
println city +" "+ districts.size()
def index = 0
for(def d : districts){
index ++
println index +" "+ d.region
if(set.contains(d.region)){
continue
}
@@ -157,8 +168,12 @@ class RegionStandardBatch extends BaseBatch{
def updateRegion = null
def flag = null
for(def key : set){
def alias = streetMap.get(key)
if((alias && alias.contains(d.region)) || (!alias && key.contains(d.region))){
def obj = streetMap.get(key)
def alias = null
if(obj && obj.alias){
alias = obj.alias
}
if((alias && alias.contains(d.region)) || (!alias && key==d.region)){
def newRegion = RegionData.formatRegion(d.city, key)
if(newRegion!=d.region){
updateRegion = newRegion
@@ -166,13 +181,26 @@ class RegionStandardBatch extends BaseBatch{
flag = "pass"
break;
}
if(obj.shape){
def inAreaFlag = InsidePolygon.isInArea(d.lng/10000000000.0, d.lat/10000000000.0, InsidePolygon.createPoint4(obj.shape.coordinates))
if(inAreaFlag){
def newRegion = obj.street
newRegion = RegionData.formatRegion(d.city, newRegion)
if(newRegion!=d.region){
updateRegion = newRegion
}
flag = "pass"
break;
}
}
}
if(updateRegion){
def udata = new BasicDBObject()
udata.put("region", updateRegion)
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name +" ÇøÓòÐÞ¸ÄΪ£º"+updateRegion
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
}else{
if(flag!="pass"){
def udata = new BasicDBObject()
@@ -402,4 +430,27 @@ class RegionStandardBatch extends BaseBatch{
return rets;
}
def updateRegionByTown(){
ExcelUtils.modify("d:\\数据\\城市与区域相同写字楼.xlsx",{d->
def town = RegionData.formatRegion(d['B'], d['F'])
if(town){
def obj = CommonDatautils.findDistrict(d['B'], town, d['E'], "office")
if(!obj){
def udata = new BasicDBObject()
udata.put("region", town)
udata.put("m_date", new Date())
println d['F']+" "+" 区域修改为:" + town
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata))
}else{
def udata = new BasicDBObject()
udata.put("m_date", new Date())
udata.put("status", 1)
println d['F']+" "+" 删除:" + town
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata))
}
}
})
}
}
@@ -16,7 +16,7 @@ class RegionUpdateBatch {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
static main(args) {
def task = new RegionUpdateBatch()
@@ -1,11 +1,13 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.bson.types.ObjectId
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
@@ -23,12 +25,15 @@ class SynchAdministrativeMappingDataBatch extends BaseBatch{
@Override
public void process(Object args) {
def indexName = "administrative_mapping"
def retMap = synchAdminiMappingTo100()
synchToEs(retMap, "192.168.0.89")
// synchToEs(retMap, "192.168.0.52")
synchToEs(retMap, "192.168.0.89", indexName)
synchToEs(retMap, "192.168.0.52", indexName)
println(" ÔËÐÐÍê³É")
}
def synchAdminiMappingTo100(){
def synchAdminiMappingTo100(city){
//删除旧的,再同步新的
ADMINISTRATIVE_MAPPING_COLL_100.remove(new BasicDBObject())
@@ -52,12 +57,11 @@ class SynchAdministrativeMappingDataBatch extends BaseBatch{
return retMap;
}
def synchToEs(retMap, host){
def synchToEs(retMap, host, indexName){
QueryBuilder queryBuilder = QueryBuilders.boolQuery()
def oldIds = new CopyOnWriteArrayList()
ElasticUtils89.executeQuery(host, "administrative_mapping", queryBuilder, { String id, doc->
ElasticUtils89.queryUpdate(host, indexName, queryBuilder, { String id, doc->
oldIds << id
def mongoObj = retMap.get(id)
def updateFieldCount = 0
@@ -65,8 +69,7 @@ class SynchAdministrativeMappingDataBatch extends BaseBatch{
if(!["id","_id","c_date","m_date","polyline_source","operator","city_simple_pinyin","prefecture_level_city"].contains(k)){
def val = doc.get(k)
if(val!=v){
// updateValMap.put(k,val)
if(v.type!="MultiPolygon"){
if(k!="shape" || (k=="shape" && v.type!="MultiPolygon")){
println k +" 修改 "+id
updateFieldCount++
doc.put(k,v)
@@ -80,14 +83,14 @@ class SynchAdministrativeMappingDataBatch extends BaseBatch{
}else{
if(!mongoObj){
println "删除:"+id +" "+doc.name
ElasticUtils.delete(id, host, "administrative_mapping")
ElasticUtils.delete(id, host, indexName)
return false
}
}
})
def addEsDatas = []
def addEsDatasMap = [:]
retMap.each{id,map->
if(!oldIds.contains(id)){
println "需新增:"+id
@@ -95,11 +98,12 @@ class SynchAdministrativeMappingDataBatch extends BaseBatch{
map.remove("m_date")
map.remove("polyline_source")
map.remove("operator")
addEsDatas << map
map.remove("_id")
addEsDatasMap.put(id, map)
}
}
if(addEsDatas){
ElasticUtils89.updateEs(addEsDatas, host, "administrative_mapping")
if(addEsDatasMap){
ElasticUtils89.updateEs(addEsDatasMap, host, indexName)
}
}
@@ -10,6 +10,7 @@ import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
/**
* 将43上的开发区同步到mongo-100, es-89, es-52
@@ -27,9 +28,10 @@ class SynchDevDataBatch extends BaseBatch{
@Override
public void process(Object args) {
def indexName = "development_zone"
def devMap = synchDevTo100()
synchDevEs(devMap, "192.168.0.89")
synchDevEs(devMap, "192.168.0.52")
synchDevEs(devMap, "192.168.0.89", indexName)
synchDevEs(devMap, "192.168.0.52", indexName)
}
def synchDevTo100(){
@@ -81,10 +83,12 @@ class SynchDevDataBatch extends BaseBatch{
}
def synchDevEs(devMap, host){
def synchDevEs(devMap, host, indexName){
QueryBuilder queryBuilder = QueryBuilders.boolQuery()
def oldIds = new CopyOnWriteArrayList()
ElasticUtils89.executeQuery(host, "development_zone", queryBuilder, { String id, doc->
ElasticUtils89.executeQuery(host, indexName, queryBuilder, { String id, doc->
oldIds << id
def mongoObj = devMap.get(id)
def updateFieldCount = 0
mongoObj.each{k,v->
@@ -92,7 +96,6 @@ class SynchDevDataBatch extends BaseBatch{
def val = doc.get(k)
if(val!=v){
println doc.name +" 修改 "+k+" "+ v+" "+id
// updateValMap.put(k,val)
updateFieldCount++
doc.put(k,v)
}
@@ -104,11 +107,32 @@ class SynchDevDataBatch extends BaseBatch{
}else{
if(!mongoObj){
println "删除:"+id +" "+doc.name
ElasticUtils.delete(id, host, "development_zone")
ElasticUtils.delete(id, host, indexName)
return false
}
}
})
def addEsDatasMap = [:]
devMap.each{id,map->
if(!oldIds.contains(id)){
println "ÐèÐÂÔö£º"+id
map.remove("create_time")
map.remove("build_date")
map.remove("bd_shape")
map.remove("img_url")
map.remove("operator")
map.remove("operate_result")
map.remove("m_date")
map.remove("township")
map.remove("block")
map.remove("_id")
addEsDatasMap.put(id, map)
}
}
if(addEsDatasMap){
ElasticUtils89.updateEs(addEsDatasMap, host, indexName)
}
}
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
@@ -219,34 +220,41 @@ class TempExportData {
}
def exportOfficeHouseCount(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\写字楼\\待核实写字楼.xlsx",{d->
ExcelUtils.multiThreadingModify("D:\\数据\\高德写字楼-房源第三方验证.xlsx",{d->
// def obj = findYuanQu(d['B'], d['D'])
// if(obj){
// d['T'] = "ǰհ´æÔÚ"
// }
// if(!d['T']){
// def obj = findYuanQu(d['B'], d['D'])
// def city = RegionData.formatCity(d['C'])
// def obj = findYuanQu(city, d['E'])
// if(obj){
// d['T'] = "前瞻存在"
// d['U'] = obj.name
// }
// }
// if(!d['T']) {
// def count = findHouseCountByCat(d['B'], d['C'], d['D'], "office_sell")
// def city = RegionData.formatCity(d['C'])
// def region = RegionData.formatRegion(city, d['D'])
//
// def count = findHouseCountByCat(city, region, d['E'], "office_sell")
// if (!count) {
// count = findHouseCountByCat(d['B'], d['C'], d['D'], "office_rent")
// count = findHouseCountByCat(city, region, d['E'], "office_rent")
// }
// d['S'] = count
// d['T'] = count
// }
if(!d['J']){
def datas = CommonDatautils.findThirdNetWorkOffices(d['B'], d['C'], d['D'])
if(!d['T']){
def city = RegionData.formatCity(d['C'])
def region = RegionData.formatRegion(city, d['D'])
def datas = CommonDatautils.findThirdNetWorkOffices(city, region, d['E'])
if(datas){
def sources = []
for(def o : datas){
sources << o.source
}
if(sources){
d['K'] = sources.join(",")
d['V'] = sources.join(",")
}
}
}
@@ -1,58 +1,57 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.lang.math.RandomUtils
def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName2("fangjia", "region_stat", mongoDbUrl)
def city = "昌江"
ExcelUtils.modify("D:\\数据\\海南\\海南省城市均价.xlsx",{d->
def date = d['B']
def start = DateUtils.getFirstDayOfMonth(date)
def end = DateUtils.getLastDayOfMonth(date)
def weekends = DateUtils.getWeekendBetweenDates(start, end)
for(def weekend : weekends){
def avg_price = d['I']
def val = random(-200,200)
def new_price = (int)(avg_price*val/10000 + avg_price)
def last_weekend = DateUtils.add(DateUtils.formatDateByPattern(weekend, "yyyy-MM-dd"), -7)
def last_price = CommonDatautils.queryCityPriceByWeekend(city, last_weekend)
def o = new BasicDBObject()
o.put("avg_price", new_price)
o.put("c_date" , new Date())
o.put("city", city)
o.put("m_date" , new Date())
o.put("weekend", weekend)
o.put("cat","城市")
o.put("moving_average" , (int)avg_price)
if(!last_price){
o.put("weekend_markup" , 0)
}else{
def weekend_markup = (int)((new_price-last_price)*100/last_price)
o.put("weekend_markup" , weekend_markup)
}
def q = new BasicDBObject("cat", "城市").append("city", city).append("weekend", weekend)
REGION_STAT_COLL.update(q, new BasicDBObject('$set', o),true, false)
}
},1)
public static final int random(int i1, int i2) {
int max = Math.max(i1, i2);
int min = Math.min(i1, i2);
return min + RandomUtils.nextInt(max - min + 1);
}
import org.bson.types.ObjectId
ObjectId objectId = new ObjectId("659c9ae92ae7732fd7d54e40");
Date creationDate = objectId.getDate();
System.out.println(creationDate);
//def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
//DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName2("fangjia", "region_stat", mongoDbUrl)
//
//
//def city = "昌江"
//ExcelUtils.modify("D:\\数据\\海南\\海南省城市均价.xlsx",{d->
// def date = d['B']
// def start = DateUtils.getFirstDayOfMonth(date)
// def end = DateUtils.getLastDayOfMonth(date)
// def weekends = DateUtils.getWeekendBetweenDates(start, end)
// for(def weekend : weekends){
// def avg_price = d['I']
// def val = random(-200,200)
// def new_price = (int)(avg_price*val/10000 + avg_price)
// def last_weekend = DateUtils.add(DateUtils.formatDateByPattern(weekend, "yyyy-MM-dd"), -7)
// def last_price = CommonDatautils.queryCityPriceByWeekend(city, last_weekend)
//
// def o = new BasicDBObject()
// o.put("avg_price", new_price)
// o.put("c_date" , new Date())
// o.put("city", city)
// o.put("m_date" , new Date())
// o.put("weekend", weekend)
// o.put("cat","城市")
// o.put("moving_average" , (int)avg_price)
// if(!last_price){
// o.put("weekend_markup" , 0)
// }else{
// def weekend_markup = (int)((new_price-last_price)*100/last_price)
// o.put("weekend_markup" , weekend_markup)
// }
//
// def q = new BasicDBObject("cat", "城市").append("city", city).append("weekend", weekend)
// REGION_STAT_COLL.update(q, new BasicDBObject('$set', o),true, false)
// }
//},1)
//
//
//public static final int random(int i1, int i2) {
// int max = Math.max(i1, i2);
// int min = Math.min(i1, i2);
// return min + RandomUtils.nextInt(max - min + 1);
//}
//
//
//
//
//
@@ -3,23 +3,22 @@ package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.BaiduMapUtils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ProvinceData
import com.mongodb.AggregationOutput
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.client.AggregateIterable
import org.bson.Document
import net.sf.json.JSONObject
import org.bson.types.ObjectId
import java.util.regex.Pattern
/**
* 乡镇街道中间表
* 包含市,区,乡镇街道别名,来源 统计局community表
@@ -45,6 +44,8 @@ class TownMiddleProcess {
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
static DBCollection POLYLINECOLL2025 = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline_2025")
def fullCityMap = CommonDatautils.queryFullNameCity()
static main(args) {
@@ -96,6 +97,12 @@ class TownMiddleProcess {
// checkCenterInTownRang()//检测乡镇街道中心点是否在围栏中
// addSpecialCityTownAlias()
// updateNewRegionPoyline()//更新新的区域围栏
updateMappingPolyline()
}
@@ -681,7 +688,7 @@ class TownMiddleProcess {
}
def regionPolyline(){
def dbcursor = REGIONPOLYLINE_COLL.find(new BasicDBObject("level","district"))
def dbcursor = POLYLINECOLL2025.find(new BasicDBObject("level","district"))
def map = [:]
for(def d : dbcursor){
def resultMap = [:]
@@ -1356,4 +1363,90 @@ class TownMiddleProcess {
keys.put("street", 1)
return ADMINISTRATIVE_MAPPING_COLL.find(q, keys).toList()
}
def updateNewRegionPoyline(){
def regionPolylineMap = regionPolyline()
def q = new BasicDBObject()
q.put("type","区域")
// q.put("city", "苏州市")
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
for(def d : dbcursor){
def shapeData = regionPolylineMap.get(d.city+"|@|"+d.district)
println d.city+" "+d.district
if(shapeData){
println "新围栏数:"+shapeData.shape.coordinates.size()
}
if(d.shape){
println "旧围栏数:"+d.shape.coordinates.size()
}
if(d.shape && shapeData && shapeData.shape!=d.shape){
def udata = new BasicDBObject()
udata.shape = shapeData.shape
udata.location = shapeData.location
udata.m_date = new Date()
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
// if(shapeData && d.shape && shapeData.shape.type=="polygon" && d.shape.type=="polygon"){
// double area = com.fangjia.script.util.GeoHelper.calculateArea(shapeData.shape.coordinates[0])
// double area1 = com.fangjia.script.util.GeoHelper.calculateArea(d.shape.coordinates[0])
// if(Math.abs(area1-area)>=1){
// println "新:"+area
// println "旧:"+area1
// def udata = new BasicDBObject()
// udata.shape = shapeData.shape
// udata.location = shapeData.location
// //DMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
// }
// }
}
}
}
def updateMappingPolyline(){
def q = new BasicDBObject()
q.put("type","区域")
q.put("shape.type","polygon")
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
for(def d : dbcursor){
updateAdministrativeMapping(d._id.toString(), d.shape)
}
}
def updateAdministrativeMapping(def id, def shape){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/administrative_mapping/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
def body = """
{
"script": {
"source": "ctx._source.shape = params.newshape",
"params":{
"newshape":{
"type":"${shape.type}",
"coordinates":${shape.coordinates}
}
}
},
"query": {
"term": {
"_id": "${id}"
}
}
}
"""
HttpRequest request = new HttpRequest()
try{
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = JSONObject.fromObject(res)
println(response)
}catch(Exception e){
println "error : "+e.getMessage()
}
}
}
@@ -31,7 +31,6 @@ import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 区域标准化
* 1.小区库乡镇街道标准化,包含小区,商铺,写字楼
@@ -629,7 +628,7 @@ class TownStandardBatch extends BaseBatch{
// //通过调取地图api接口获取乡镇街道
// if(!saveObj.township){
// def gaodeMap = BaiduMapUtils.geocoding(location.lat+","+location.lon)
def gaodeMap = BaiduMapUtils.geocoding(location.lat+","+location.lon)
// if(gaodeMap && gaodeMap.addressComponent) {
// def addressComponent = gaodeMap.addressComponent
// if(addressComponent){
@@ -1,16 +1,9 @@
package com.fangjia.script.gj.district
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.addrepo.InvokeHttpRequest
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.util.HttpRequestUtil
import com.mongodb.DBObject
import net.sf.json.JSONArray
import net.sf.json.JSONObject
import org.apache.commons.collections.CollectionUtils
import org.apache.commons.lang.StringUtils
class FangjiaClientUtils {
@@ -9,7 +9,7 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def city = "³£ÖÝ"
@@ -14,7 +14,7 @@ import org.apache.lucene.queryparser.classic.QueryParser
class EsMonthHouseStat {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
static main(args) {
@@ -38,7 +38,7 @@ class FjwBigdataDealBatch extends BaseBatch{
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
@@ -29,7 +29,7 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07",
@@ -27,7 +27,7 @@ class GuangfaBlankSelectDistrict {
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
static main(args) {
def task = new GuangfaBlankSelectDistrict();
@@ -31,7 +31,7 @@ import java.util.concurrent.atomic.AtomicLong
public class BeiKeCrawlByChrome {
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
WebDriver driver = null;
@@ -118,7 +118,7 @@ public class BeiKeCrawlByChrome {
}
def crawlData(driver, citys){
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def runedCitys = FileUtils.readTxt("d:\\数据\\成交抓取\\贝壳已抓城市.txt")
@@ -15,7 +15,7 @@ import java.util.regex.Pattern
class ExportPaimai {
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
static main(args) {
@@ -31,7 +31,7 @@ import java.util.concurrent.atomic.AtomicLong
public class LianjiaCrawlByChrome {
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
WebDriver driver = null;
@@ -189,7 +189,7 @@ public class LianjiaCrawlByChrome {
}
def crawlData(driver, cityList){
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def runedCitys = FileUtils.readTxt("d:\\数据\\成交抓取\\链家已抓城市.txt")
@@ -28,7 +28,7 @@ class PaimaiDataSend {
DBCollection jd_coll = MongoUtils.getCollectionByName("fangjia_crawl", "fapai_jd_detail", "192.168.0.235", 27018)
DBCollection rmfysszc_coll = MongoUtils.getCollectionByName("fangjia_crawl", "fapai_rmfysszc_detail", "192.168.0.235", 27018)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
@@ -26,7 +26,7 @@ class PaimaiTest {
DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai")
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
@@ -30,7 +30,7 @@ class AddPriceTest {
def pgsqlInstance235 = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
public static final DBCollection confirm_price_date = MongoUtils.getCollectionByName("fangjia", "confirm_price_data", "mongo.db.fangjia.com")
static main(args) {
@@ -230,6 +230,8 @@ class AddPriceTest {
if(newPriceMap){
d['G'] = newPriceMap.avg_price
d['H'] = newPriceMap.month
d['H'] = map.calc_type
// d['AM'] = map.credit_value
}
}
})
@@ -13,8 +13,8 @@ import java.util.concurrent.atomic.AtomicInteger
class ExportToDealPrice {
DBCollection DEAL_PRICE_MATCH_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price_match", "192.168.0.43")
DBCollection DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43")
DBCollection DEAL_PRICE_MATCH_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price_match", "mongo.db.fangjia.com")
DBCollection DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
DBCollection TRADE_MATCH_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "trade_match_price", "192.168.0.136")
@@ -14,7 +14,7 @@ import java.util.regex.Pattern
class FzBlankTest {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection FZ_ADDR_COLL = MongoUtils.getCollectionByName2("business", "fz_district_address", lineUrl)
static main(args) {
@@ -20,7 +20,7 @@ import java.util.regex.Pattern
class JiangShuBlankDataStat {
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
static main(args) {
def task = new JiangShuBlankDataStat()
@@ -12,7 +12,7 @@ import com.mongodb.DBCollection
class TestArroundPrice {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection DIST_DATA_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_analy_data", lineUrl)
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
@@ -15,7 +15,7 @@ import java.util.concurrent.atomic.AtomicLong
class StreetProcess {
final DBCollection TOWNSHIP = MongoUtils.getCollectionByName("fangjia", "township", "192.168.0.43")
final DBCollection TOWNSHIP = MongoUtils.getCollectionByName("fangjia", "township", "mongo.db.fangjia.com")
public static final def MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
public static final DBCollection TOWNSHIP100 = MongoUtils.getCollectionByName2("fangjia", "township", MONGODB_URL)
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.xiamen
import com.fangjia.script.util.ElasticUtils
import org.apache.http.HttpHost
import org.apache.http.auth.AuthScope
import org.apache.http.auth.UsernamePasswordCredentials
@@ -7,8 +8,10 @@ import org.apache.http.client.CredentialsProvider
import org.apache.http.client.config.RequestConfig
import org.apache.http.impl.client.BasicCredentialsProvider
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder
import org.apache.solr.client.solrj.request.UpdateRequest
import org.elasticsearch.action.ActionListener
import org.elasticsearch.action.bulk.BackoffPolicy
import org.elasticsearch.action.bulk.BulkItemResponse
import org.elasticsearch.action.bulk.BulkProcessor
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
@@ -17,11 +20,13 @@ import org.elasticsearch.action.search.ClearScrollRequest
import org.elasticsearch.action.search.SearchRequest
import org.elasticsearch.action.search.SearchResponse
import org.elasticsearch.action.search.SearchScrollRequest
import org.elasticsearch.action.support.WriteRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.client.RestClient
import org.elasticsearch.client.RestClientBuilder
import org.elasticsearch.client.RestHighLevelClient
import org.elasticsearch.common.unit.TimeValue
import org.elasticsearch.common.xcontent.XContentType
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.search.Scroll
import org.elasticsearch.search.SearchHit
@@ -160,8 +165,17 @@ class ElasticUtils89 {
}
@Override
void afterBulk(long executionId, BulkRequest request,
BulkResponse response) {
public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
if (response.hasFailures()) {
for (BulkItemResponse itemResponse : response.getItems()) {
if (itemResponse.isFailed()) {
// 处理失败项
String failureMessage = itemResponse.getFailureMessage();
println failureMessage
// 日志记录、告警或重试
}
}
}
}
@Override
@@ -205,6 +219,7 @@ class ElasticUtils89 {
}
BulkProcessor bulkProcessor = createBulkProcessor(host, client)
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.scroll(scroll);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
@@ -221,7 +236,7 @@ class ElasticUtils89 {
// executorService.submit(new Runnable() {
// @Override
// void run() {
try{
// try{
if(closure.getMaximumNumberOfParameters() == 1){
def update = closure.call(hit.getSourceAsMap())
@@ -232,14 +247,16 @@ class ElasticUtils89 {
def update = closure.call(hit.getId(), hit.getSourceAsMap())
if(update){
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
}
}
}catch (Exception e){
e.printStackTrace()
}
// }catch (Exception e){
// e.printStackTrace()
// }
// }
// })
}
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
@@ -256,6 +273,120 @@ class ElasticUtils89 {
client.close()
}
static queryUpdate(String host, String indexName, QueryBuilder qb, Closure closure){
RestHighLevelClient client = null
if(host == "192.168.0.89"){
client = getClient(host);
}else if(host == "192.168.0.52"){
client = getClientOther(host);
}
BulkProcessor bulkProcessor = createBulkProcessor(host, client)
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(30))
// 2. 初始化滚动查询
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.scroll(scroll);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(qb);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
String scrollId = searchResponse.getScrollId();
SearchHit[] hits = searchResponse.getHits().getHits();
// 3. 批量操作容器
BulkRequest bulkRequest = new BulkRequest();
// 4. 处理滚动结果
while (hits.length > 0){
for (SearchHit hit : hits) {
// 示例修改逻辑:当满足条件时更新文档
if(closure.getMaximumNumberOfParameters() == 2){
def update = closure.call(hit.getId(), hit.getSourceAsMap())
if(update){
bulkRequest.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
}
}
// 批量提交控制
if (bulkRequest.numberOfActions() >= 1) {
bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
System.err.println("批量操作部分失败: " + bulkResponse.buildFailureMessage());
// 这里可以添加重试逻辑
}
bulkRequest = new BulkRequest(); // 重置容器
}
}
// 5. 获取下一批滚动结果
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId).scroll(TimeValue.timeValueMinutes(5));
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
scrollId = searchResponse.getScrollId();
hits = searchResponse.getHits().getHits();
}
// 6. 提交剩余批量操作
if (bulkRequest.numberOfActions() > 0) {
BulkResponse bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
System.err.println("批量操作部分失败: " + bulkResponse.buildFailureMessage());
}
}
// 7. 清理滚动上下文
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.addScrollId(scrollId);
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
bulkProcessor.close()
bulkProcessor.awaitClose(1, TimeUnit.DAYS)
client.close()
}
static updateEs(addEsDatasMap, host, indexName){
BulkRequest request = new BulkRequest();
int processedCount = 0;
def client = null
if(host && host == "192.168.0.89"){
client = ElasticUtils89.getClient(host)
}else if(host && host == "192.168.0.52"){
client = ElasticUtils.getClient(host)
}
addEsDatasMap.each{id, map->
request.add(new IndexRequest(indexName, "_doc", id).source(map, XContentType.JSON));
if (request.numberOfActions() >= 100) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents "+host
}
request = new BulkRequest();
}
}
if (request.numberOfActions() > 0) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents "+host
}
}
}
static def executeSimpleQuery(String host, String indexName, QueryBuilder qb, includes){
@@ -27,9 +27,9 @@ class NetworkDistanceBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
final DBCollection NETWORK_COLL = MongoUtils.getCollectionByName("business", "xm_network", "192.168.0.43")
final DBCollection NETWORK_COLL = MongoUtils.getCollectionByName("business", "xm_network", "mongo.db.fangjia.com")
final DBCollection MAP_NETWORK_COLL = MongoUtils.getCollectionByName("business", "xm_network_map", "192.168.0.43")
final DBCollection MAP_NETWORK_COLL = MongoUtils.getCollectionByName("business", "xm_network_map", "mongo.db.fangjia.com")
static main(args) {
runBatch(new NetworkDistanceBatch(), args)
@@ -78,8 +78,6 @@ class SeaweedDataSynch extends BaseBatch{
@Override
public void process(Object args) {
run()
// println "全国district, shop, office同步完成"
//民政部编码上传
@@ -167,9 +165,9 @@ class SeaweedDataSynch extends BaseBatch{
def shops = updating(c, "shop")
updateEs(c, shops, "shop", "192.168.0.89")
updateEs(c, offices, "shop", "192.168.0.52")
updateEs(c, shops, "shop", "192.168.0.52")
println index +" "+ c + " 完成"
println index +" "+ c + " 完成 "+new Date().toString()
}
}
@@ -245,6 +243,7 @@ class SeaweedDataSynch extends BaseBatch{
}
if(!esObj || (esObj && diffMap)){
map.put("id", d._id.toString())
map.put("status", 0) //因为判断是否有值,0会返回false,所在这里设置默认值,在89和52上的都是有效数据
return map
}
return null
@@ -20,10 +20,10 @@ import java.util.regex.Pattern
class ConsumptionRatingBatch extends BaseBatch {
def weekend = DateUtil.truncateDay(DateUtil.getFirstDayOfWeek(new Date()))
public static
final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43")
final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
public static
final DBCollection CONSUMPTION_GRADE_COLL = MongoUtils.getCollectionByName("fangjia_stat", "consumption_grade", "192.168.0.85")
final DBCollection CONSUMPTION_GRADE_COLL = MongoUtils.getCollectionByName("fangjia_stat", "consumption_grade", "mongo.db.fangjia.com")
final DBCollection AMAP_POI_New_COLL = MongoUtils.getCollectionByName("m_dianping", "dianping_9", "114.80.150.196", 27777)
@@ -310,7 +310,7 @@ class SMSOutgoingListener extends BaseBatch {
if(StringUtils.isNotBlank(checkCode)){
JSONObject con = new JSONObject()
con.put("code", checkCode);
return AliyunSmsClient.sendSms(mobile, "SMS_482955117", con.toString())
return AliyunSmsClient.sendSms(mobile, "SMS_483345375", con.toString())
}
}
int resCode = SingletonClient.getClient().sendSMS([mobile] as String[],content, 5)
@@ -52,7 +52,7 @@ class AliyunSmsClient {
}
public static void main(String[] args) {
sendSms("13916076378", "SMS_135220034", '{"code":12345}')
sendSms("15021384995", "SMS_482915107", '{"code":12345}')
}
@@ -30,7 +30,7 @@ import java.sql.SQLException
class HousingTransactionExportBatch extends BaseBatch {
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
static DBCollection _db_esf_transaction = MongoUtils.getCollectionByName("vip_site", "esf_transaction", "192.168.0.235", 27018);
static MongoCollection<Document> PAI_MAI = Mongo3Utils.getDbCollection(Constants.FANGJIA_MONGO_URI, "fangjia_crawl", "pai_mai")
@@ -23,7 +23,7 @@ import java.util.regex.Pattern
class CqfdcMatch {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", lineUrl)
static DBCollection loudongFdcConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong_cqfdc", "192.168.0.235", 27018)
@@ -53,8 +53,8 @@ def pcmap = [:]
provinceList.remove("广西省")
provinceList.add("广西壮族自治区")
DSCollection seabird = new DataStore("192.168.0.43", 28017).getCollection("fangjia_base.baidu_place_poi2")
DBCollection m43 = MongoUtils.getCollectionByName("fangjia_base", "baidu_place_poi2", "192.168.0.43", 28017)
DSCollection seabird = new DataStore("mongo.db.fangjia.com", 28017).getCollection("fangjia_base.baidu_place_poi2")
DBCollection m43 = MongoUtils.getCollectionByName("fangjia_base", "baidu_place_poi2", "mongo.db.fangjia.com", 28017)
//def type3Not = ["学校内部设施", "进港", "出港", "候船室", "住宿服务相关"]
//
//def type2Not = ["学校", "专科医院", "综合医院", "诊所", "银行", "自动提款机", "便民商店/便利店", "家电电子卖场", "超级市场", "商场", "美容美发店", "地铁站", "港口码头", "住宿服务"]
@@ -16,9 +16,9 @@ import com.mongodb.DBCollection
//@BatchOwner(Watcher.MaJinTao)
class CommunityHouseSampleBatch extends BaseBatch {
DBCollection SEAWEED = MongoUtils.getCollectionByName("fangjia", "seaweed", "192.168.0.43", 28017)
DBCollection STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 28017)
DBCollection HOUSE_CASE = MongoUtils.getCollectionByName("business", "house_case", "192.168.0.43", 28017)
DBCollection SEAWEED = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com", 28017)
DBCollection STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com", 28017)
DBCollection HOUSE_CASE = MongoUtils.getCollectionByName("business", "house_case", "mongo.db.fangjia.com", 28017)
DSCollection SEABIRD = new DataStore("seabird.db.fangjia.com").getCollection("fangjia.seabird")
static void main(String[] args) {
@@ -160,13 +160,13 @@ import com.mongodb.DBCollection
@BatchOwner(Watcher.MaJinTao)
class CommunityHouseSampleBatch extends BaseBatch {
DBCollection SEAWEED = MongoUtils.getCollectionByName("fangjia", "seaweed", "192.168.0.43", 28017)
DBCollection SEAWEED = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com", 28017)
DBCollection STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 28017)
DBCollection STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com", 28017)
DBCollection HOUSE_CASE = MongoUtils.getCollectionByName("business", "house_case", "192.168.0.43", 28017)
DBCollection HOUSE_CASE = MongoUtils.getCollectionByName("business", "house_case", "mongo.db.fangjia.com", 28017)
// DBCollection SEABIRD = MongoUtils.getCollectionByName("fangjia", "seabird", "192.168.0.43", 28017)
// DBCollection SEABIRD = MongoUtils.getCollectionByName("fangjia", "seabird", "1mongo.db.fangjia.com", 28017)
DSCollection SEABIRD = new DataStore("seabird.db.fangjia.com").getCollection("fangjia.seabird")

Some files were not shown because too many files have changed in this diff Show More