Merge remote-tracking branch 'origin/master'
This commit is contained in:
+9
@@ -0,0 +1,9 @@
|
|||||||
|
package com.fangjia.dal.mongo.open.dao;
|
||||||
|
|
||||||
|
import com.fangjia.dal.MongoDao;
|
||||||
|
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
|
||||||
|
import com.fangjia.dal.mongo.open.model.CommunityPoi;
|
||||||
|
|
||||||
|
public interface CommunityBasePriceDao extends MongoDao<CommunityBasePrice> {
|
||||||
|
|
||||||
|
}
|
||||||
+10
@@ -0,0 +1,10 @@
|
|||||||
|
package com.fangjia.dal.mongo.open.dao.impl;
|
||||||
|
|
||||||
|
import com.fangjia.dal.MongoDaoImpl;
|
||||||
|
import com.fangjia.dal.mongo.open.dao.CommunityBasePriceDao;
|
||||||
|
import com.fangjia.dal.mongo.open.dao.CommunityPoiDao;
|
||||||
|
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
|
||||||
|
import com.fangjia.dal.mongo.open.model.CommunityPoi;
|
||||||
|
|
||||||
|
public class CommunityBasePriceDaoImpl extends MongoDaoImpl<CommunityBasePrice> implements CommunityBasePriceDao {
|
||||||
|
}
|
||||||
+74
@@ -0,0 +1,74 @@
|
|||||||
|
package com.fangjia.dal.mongo.open.model;
|
||||||
|
|
||||||
|
import com.fangjia.dal.MongoEntity;
|
||||||
|
|
||||||
|
import javax.persistence.Column;
|
||||||
|
import javax.persistence.Entity;
|
||||||
|
import javax.persistence.Id;
|
||||||
|
import javax.persistence.Table;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Ð¡Çø½¨ÐÐÖÃÐŶÈÐÅÏ¢
|
||||||
|
*
|
||||||
|
* @author shenjian@fangjia.com
|
||||||
|
*/
|
||||||
|
@Entity
|
||||||
|
@Table(name = "business.community_base_price_gj")
|
||||||
|
public class CommunityBasePrice implements MongoEntity {
|
||||||
|
|
||||||
|
private static final long serialVersionUID = -8010010773207622878L;
|
||||||
|
|
||||||
|
private String id;
|
||||||
|
private String city;
|
||||||
|
private String region;
|
||||||
|
private String name;
|
||||||
|
private Double credit;
|
||||||
|
|
||||||
|
@Id
|
||||||
|
@Column(name = "_id")
|
||||||
|
@Override
|
||||||
|
public String getId() {
|
||||||
|
return id;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setId(String id) {
|
||||||
|
this.id = id;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Column(name = "city")
|
||||||
|
public String getCity() {
|
||||||
|
return city;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setCity(String city) {
|
||||||
|
this.city = city;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Column(name = "region")
|
||||||
|
public String getRegion() {
|
||||||
|
return region;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setRegion(String region) {
|
||||||
|
this.region = region;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Column(name = "name")
|
||||||
|
public String getName() {
|
||||||
|
return name;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setName(String name) {
|
||||||
|
this.name = name;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Column(name = "credit")
|
||||||
|
public Double getCredit() {
|
||||||
|
return credit;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setCredit(Double credit) {
|
||||||
|
this.credit = credit;
|
||||||
|
}
|
||||||
|
}
|
||||||
+144
-33
@@ -31,6 +31,7 @@ import javax.validation.Valid;
|
|||||||
import com.fangjia.core.util.*;
|
import com.fangjia.core.util.*;
|
||||||
import com.fangjia.dal.*;
|
import com.fangjia.dal.*;
|
||||||
import com.fangjia.dal.mongo.model.*;
|
import com.fangjia.dal.mongo.model.*;
|
||||||
|
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
|
||||||
import com.fangjia.dal.mongo.open.model.CommunityPoi;
|
import com.fangjia.dal.mongo.open.model.CommunityPoi;
|
||||||
import com.fangjia.dal.seawater.dao.SeawaterDao;
|
import com.fangjia.dal.seawater.dao.SeawaterDao;
|
||||||
import com.fangjia.dal.seawater.model.Seawater;
|
import com.fangjia.dal.seawater.model.Seawater;
|
||||||
@@ -177,7 +178,9 @@ public class PropertyController {
|
|||||||
private SeawaterDao seawaterDao;
|
private SeawaterDao seawaterDao;
|
||||||
private SeawaterService seawaterService;
|
private SeawaterService seawaterService;
|
||||||
private SqmetreRentService sqmetreRentService;
|
private SqmetreRentService sqmetreRentService;
|
||||||
private CommunityPoiServiceImpl communityPoiService;
|
private CommunityPoiService communityPoiService;
|
||||||
|
private CommunityBasePriceService communityBasePriceService;
|
||||||
|
private EvaluateFactorService evaluateFactorService;
|
||||||
|
|
||||||
private FileDao fileDao = FileDao.getInstance();
|
private FileDao fileDao = FileDao.getInstance();
|
||||||
private static final int DEFAULT_PAGE_SIZE = 20;
|
private static final int DEFAULT_PAGE_SIZE = 20;
|
||||||
@@ -393,10 +396,31 @@ public class PropertyController {
|
|||||||
put("5000", "five");// 五线
|
put("5000", "five");// 五线
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
/**
|
||||||
|
* 第三方均价映射
|
||||||
|
*/
|
||||||
|
private static final Map<String, String> OTHER_PRICE_MAP = new HashMap<String, String>() {
|
||||||
|
{
|
||||||
|
put("zhuge_price", "诸葛找房");
|
||||||
|
put("anjuke_price", "安居客");
|
||||||
|
put("fang_price", "房天下");
|
||||||
|
put("lianjia_price", "链家");
|
||||||
|
put("cityhouse_price", "禧泰数据");
|
||||||
|
put("maitian_price", "麦田");
|
||||||
|
put("ke_price", "贝壳");
|
||||||
|
put("taofang_price", "淘房365");
|
||||||
|
put("zhongyuan_price", "中原");
|
||||||
|
put("tbs_price", "兔博士");
|
||||||
|
put("qfang_price", "Q房");
|
||||||
|
put("fangtan_price", "房探");
|
||||||
|
put("fangduoduo_price", "房多多");
|
||||||
|
put("wiwj_price", "我爱我家");
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 利用百度API把GPS转换成Google经纬度
|
* 利用百度API把GPS转换成Google经纬度
|
||||||
*
|
*
|
||||||
* @param lng
|
* @param lng
|
||||||
* @param lat
|
* @param lat
|
||||||
* @return
|
* @return
|
||||||
@@ -418,7 +442,7 @@ public class PropertyController {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* 利用算法模板把GPS转换成Google经纬度
|
* 利用算法模板把GPS转换成Google经纬度
|
||||||
*
|
*
|
||||||
* @param lng
|
* @param lng
|
||||||
* @param lat
|
* @param lat
|
||||||
* @return
|
* @return
|
||||||
@@ -610,7 +634,7 @@ public class PropertyController {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* 判断小区是否有经纬度
|
* 判断小区是否有经纬度
|
||||||
*
|
*
|
||||||
* @param district
|
* @param district
|
||||||
* @return
|
* @return
|
||||||
*/
|
*/
|
||||||
@@ -621,7 +645,7 @@ public class PropertyController {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* 调用匹配接口匹配小区
|
* 调用匹配接口匹配小区
|
||||||
*
|
*
|
||||||
* @param city
|
* @param city
|
||||||
* @param address
|
* @param address
|
||||||
* @param region
|
* @param region
|
||||||
@@ -995,6 +1019,32 @@ public class PropertyController {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 调整面积不同面积价格
|
||||||
|
try {
|
||||||
|
DistrictStatBo districtStatBo = districtStatService.findDistrictStat(new DistrictKey(district));
|
||||||
|
int orgAvgPrice = 0;
|
||||||
|
if (districtStatBo != null) {
|
||||||
|
if (StringUtils.isNotBlank(district.getEstateType2()) && district.getEstateType2().contains("别墅") && districtStatBo.getVillaDistrictStat() != null) {
|
||||||
|
orgAvgPrice = districtStatBo.getVillaDistrictStat().getAvgPrice();
|
||||||
|
} else if (districtStatBo.getDistrictStat() != null) {
|
||||||
|
orgAvgPrice = districtStatBo.getDistrictStat().getAvgPrice();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) {
|
||||||
|
int factorPrice = evaluation.getAvgPrice();
|
||||||
|
int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice);
|
||||||
|
Integer minPrice = (int)(evaluation.getAvgPrice() * 0.95);
|
||||||
|
Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.05);
|
||||||
|
// 偏离度超过5%不进行调整
|
||||||
|
if (minPrice <= avgPrice && avgPrice <= maxPrice) {
|
||||||
|
evaluation.setAvgPrice(avgPrice);
|
||||||
|
evaluation.setBigTotalPrice((long) (avgPrice * evaluation.getArea()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (Exception ex) {
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
if (evaluation.getTotalPrice() != null && evaluation.getTotalPrice() != 0
|
if (evaluation.getTotalPrice() != null && evaluation.getTotalPrice() != 0
|
||||||
&& (cityPriceCredit == null || evaluation.getCredit() >= cityPriceCredit)) {
|
&& (cityPriceCredit == null || evaluation.getCredit() >= cityPriceCredit)) {
|
||||||
Map<String, Object> districtPriceMap = new HashMap<String, Object>();
|
Map<String, Object> districtPriceMap = new HashMap<String, Object>();
|
||||||
@@ -1165,6 +1215,31 @@ public class PropertyController {
|
|||||||
evaluationService.remove(evaluation.getId());
|
evaluationService.remove(evaluation.getId());
|
||||||
mav.setModel(BasicModel.buildOKModel().setMsg("暂时不支持该小区的估价").markNoData());
|
mav.setModel(BasicModel.buildOKModel().setMsg("暂时不支持该小区的估价").markNoData());
|
||||||
return mav;
|
return mav;
|
||||||
|
}
|
||||||
|
// 调整面积不同面积价格
|
||||||
|
try {
|
||||||
|
DistrictStatBo districtStatBo = districtStatService.findDistrictStat(new DistrictKey(district));
|
||||||
|
int orgAvgPrice = 0;
|
||||||
|
if (districtStatBo != null) {
|
||||||
|
if (StringUtils.isNotBlank(district.getEstateType2()) && district.getEstateType2().contains("别墅") && districtStatBo.getVillaDistrictStat() != null) {
|
||||||
|
orgAvgPrice = districtStatBo.getVillaDistrictStat().getAvgPrice();
|
||||||
|
} else if (districtStatBo.getDistrictStat() != null) {
|
||||||
|
orgAvgPrice = districtStatBo.getDistrictStat().getAvgPrice();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) {
|
||||||
|
int factorPrice = evaluation.getAvgPrice();
|
||||||
|
int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice);
|
||||||
|
Integer minPrice = (int)(evaluation.getAvgPrice() * 0.95);
|
||||||
|
Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.05);
|
||||||
|
// 偏离度超过5%不进行调整
|
||||||
|
if (minPrice <= avgPrice && avgPrice <= maxPrice) {
|
||||||
|
evaluation.setAvgPrice(avgPrice);
|
||||||
|
evaluation.setBigTotalPrice((long) (avgPrice * evaluation.getArea()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (Exception ex) {
|
||||||
|
|
||||||
}
|
}
|
||||||
Map<String, Object> districtPriceMap = new LinkedHashMap<String, Object>();
|
Map<String, Object> districtPriceMap = new LinkedHashMap<String, Object>();
|
||||||
districtPriceMap.put("city", district.getCity());
|
districtPriceMap.put("city", district.getCity());
|
||||||
@@ -1179,6 +1254,13 @@ public class PropertyController {
|
|||||||
// }
|
// }
|
||||||
districtPriceMap.put("totalPrice", evaluation.getBigTotalPrice());
|
districtPriceMap.put("totalPrice", evaluation.getBigTotalPrice());
|
||||||
districtPriceMap.put("unitPrice", evaluation.getAvgPrice());
|
districtPriceMap.put("unitPrice", evaluation.getAvgPrice());
|
||||||
|
// 新增置信度
|
||||||
|
CommunityBasePrice communityBasePrice = communityBasePriceService.getById(district.getId());
|
||||||
|
if (communityBasePrice != null) {
|
||||||
|
districtPriceMap.put("credit", communityBasePrice.getCredit());
|
||||||
|
} else {
|
||||||
|
districtPriceMap.put("credit", evaluation.getCredit());
|
||||||
|
}
|
||||||
List<String> names = new ArrayList<String>();
|
List<String> names = new ArrayList<String>();
|
||||||
names.add(district.getName());
|
names.add(district.getName());
|
||||||
if(district.getAlias() != null){
|
if(district.getAlias() != null){
|
||||||
@@ -1190,6 +1272,31 @@ public class PropertyController {
|
|||||||
}
|
}
|
||||||
Integer minPrice = (int)(evaluation.getAvgPrice() * 0.8);
|
Integer minPrice = (int)(evaluation.getAvgPrice() * 0.8);
|
||||||
Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.2);
|
Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.2);
|
||||||
|
Integer min10Price = (int)(evaluation.getAvgPrice() * 0.9);
|
||||||
|
Integer max10Price = (int)(evaluation.getAvgPrice() * 1.1);
|
||||||
|
// 新增第三方均价
|
||||||
|
try {
|
||||||
|
double dCredit = 0.61;
|
||||||
|
Map<String, Object> rMap = getLastEvaluationPrice(district, evaluation, dCredit);
|
||||||
|
if (null != rMap) {
|
||||||
|
if (rMap.containsKey("other_price")) {
|
||||||
|
Map<String, Integer> otherPriceMap = (Map<String, Integer>) rMap.get("other_price");
|
||||||
|
Map<String, Integer> priceResultMap = new HashMap<String, Integer>();
|
||||||
|
for (String key : otherPriceMap.keySet()) {
|
||||||
|
Integer price = otherPriceMap.get(key);
|
||||||
|
// 剔除偏差大于10的
|
||||||
|
if (min10Price <= price && max10Price >= price) {
|
||||||
|
if (OTHER_PRICE_MAP.containsKey(key)) {
|
||||||
|
priceResultMap.put(OTHER_PRICE_MAP.get(key), price);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
districtPriceMap.put("otherPrice", priceResultMap);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (Exception ex) {
|
||||||
|
|
||||||
|
}
|
||||||
List<Seawater> houses = seawaterDao.findHouses(district.getCity(), district.getRegion(), names,minPrice, maxPrice, 20);
|
List<Seawater> houses = seawaterDao.findHouses(district.getCity(), district.getRegion(), names,minPrice, maxPrice, 20);
|
||||||
if(houses.size() > 3){
|
if(houses.size() > 3){
|
||||||
districtPriceMap.put("calcType", 1);
|
districtPriceMap.put("calcType", 1);
|
||||||
@@ -1491,7 +1598,7 @@ public class PropertyController {
|
|||||||
return mav;
|
return mav;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 四川锦程 定制接口 多建筑年代
|
* 四川锦程 定制接口 多建筑年代
|
||||||
* @param model
|
* @param model
|
||||||
@@ -1603,7 +1710,7 @@ public class PropertyController {
|
|||||||
districtPriceMap.put("buildYear", matcher3.group());
|
districtPriceMap.put("buildYear", matcher3.group());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// 根据城市等 级设置D级可信度
|
// 根据城市等 级设置D级可信度
|
||||||
ApiUserAuthority apiUserAuthority = apiUserAuthorityService
|
ApiUserAuthority apiUserAuthority = apiUserAuthorityService
|
||||||
@@ -2116,7 +2223,7 @@ public class PropertyController {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* 小区租金查询接口
|
* 小区租金查询接口
|
||||||
*
|
*
|
||||||
* @param model
|
* @param model
|
||||||
* @return
|
* @return
|
||||||
*/
|
*/
|
||||||
@@ -2595,7 +2702,7 @@ public class PropertyController {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* 批量获取小区的均价,均价走势数据
|
* 批量获取小区的均价,均价走势数据
|
||||||
*
|
*
|
||||||
* @param request
|
* @param request
|
||||||
* @return
|
* @return
|
||||||
*/
|
*/
|
||||||
@@ -2737,7 +2844,7 @@ public class PropertyController {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* 批量获取小区的配套数据,最多12个小区
|
* 批量获取小区的配套数据,最多12个小区
|
||||||
*
|
*
|
||||||
* @param request
|
* @param request
|
||||||
* @return
|
* @return
|
||||||
*/
|
*/
|
||||||
@@ -2882,7 +2989,7 @@ public class PropertyController {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* 获取JSON参数中的num参数值
|
* 获取JSON参数中的num参数值
|
||||||
*
|
*
|
||||||
* @param jo
|
* @param jo
|
||||||
* @return
|
* @return
|
||||||
*/
|
*/
|
||||||
@@ -2907,7 +3014,7 @@ public class PropertyController {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* 获取JSON参数中的radius值
|
* 获取JSON参数中的radius值
|
||||||
*
|
*
|
||||||
* @param jo
|
* @param jo
|
||||||
* @return
|
* @return
|
||||||
*/
|
*/
|
||||||
@@ -3715,7 +3822,7 @@ public class PropertyController {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* 解析出关键字中的小区名
|
* 解析出关键字中的小区名
|
||||||
*
|
*
|
||||||
* @param keyword
|
* @param keyword
|
||||||
* @return
|
* @return
|
||||||
*/
|
*/
|
||||||
@@ -3832,7 +3939,7 @@ public class PropertyController {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* 小区统计分析接口
|
* 小区统计分析接口
|
||||||
*
|
*
|
||||||
* @param model
|
* @param model
|
||||||
* @return
|
* @return
|
||||||
*/
|
*/
|
||||||
@@ -3917,7 +4024,7 @@ public class PropertyController {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* 小区交通接口
|
* 小区交通接口
|
||||||
*
|
*
|
||||||
* @param model
|
* @param model
|
||||||
* @return
|
* @return
|
||||||
*/
|
*/
|
||||||
@@ -3975,7 +4082,7 @@ public class PropertyController {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* 小区周边接口
|
* 小区周边接口
|
||||||
*
|
*
|
||||||
* @param model
|
* @param model
|
||||||
* @return
|
* @return
|
||||||
*/
|
*/
|
||||||
@@ -4034,7 +4141,7 @@ public class PropertyController {
|
|||||||
/**
|
/**
|
||||||
* 相同小区相似户型房源挂牌量接口<br>
|
* 相同小区相似户型房源挂牌量接口<br>
|
||||||
* 获取该小区的满足指定相似条件的挂牌房源量 条件说明: 1.输入面积-5<=房源面积<=输入面积+5 2.上个月发布的房源
|
* 获取该小区的满足指定相似条件的挂牌房源量 条件说明: 1.输入面积-5<=房源面积<=输入面积+5 2.上个月发布的房源
|
||||||
*
|
*
|
||||||
* @author yinjihuan
|
* @author yinjihuan
|
||||||
* @date 2016年9月1日
|
* @date 2016年9月1日
|
||||||
* @param city
|
* @param city
|
||||||
@@ -4074,7 +4181,7 @@ public class PropertyController {
|
|||||||
/**
|
/**
|
||||||
* 周边小区相似户型挂牌量接口<br>
|
* 周边小区相似户型挂牌量接口<br>
|
||||||
* 获取该小区的满足指定相似条件的挂牌房源量 条件说明: 1.输入面积-5<=房源面积<=输入面积+5 2.上个月发布的房源
|
* 获取该小区的满足指定相似条件的挂牌房源量 条件说明: 1.输入面积-5<=房源面积<=输入面积+5 2.上个月发布的房源
|
||||||
*
|
*
|
||||||
* @author yinjihuan
|
* @author yinjihuan
|
||||||
* @date 2016年9月1日
|
* @date 2016年9月1日
|
||||||
* @param city
|
* @param city
|
||||||
@@ -4125,7 +4232,7 @@ public class PropertyController {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* 小区周边一公里均价
|
* 小区周边一公里均价
|
||||||
*
|
*
|
||||||
* @author wenyan
|
* @author wenyan
|
||||||
* @param model
|
* @param model
|
||||||
* @return
|
* @return
|
||||||
@@ -4148,7 +4255,7 @@ public class PropertyController {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* 相同小区相似户型租金范围查询接口
|
* 相同小区相似户型租金范围查询接口
|
||||||
*
|
*
|
||||||
* @param model
|
* @param model
|
||||||
* @return
|
* @return
|
||||||
*/
|
*/
|
||||||
@@ -4191,7 +4298,7 @@ public class PropertyController {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* 周边小区相似户型租金范围查询接口
|
* 周边小区相似户型租金范围查询接口
|
||||||
*
|
*
|
||||||
* @param model
|
* @param model
|
||||||
* @return
|
* @return
|
||||||
*/
|
*/
|
||||||
@@ -4254,7 +4361,7 @@ public class PropertyController {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* 室号查询接口
|
* 室号查询接口
|
||||||
*
|
*
|
||||||
* @param model
|
* @param model
|
||||||
* @return
|
* @return
|
||||||
*/
|
*/
|
||||||
@@ -4290,7 +4397,7 @@ public class PropertyController {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* 楼栋楼层信息查询接口
|
* 楼栋楼层信息查询接口
|
||||||
*
|
*
|
||||||
* @param model
|
* @param model
|
||||||
* @return
|
* @return
|
||||||
*/
|
*/
|
||||||
@@ -4315,7 +4422,7 @@ public class PropertyController {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* 楼栋查询接口
|
* 楼栋查询接口
|
||||||
*
|
*
|
||||||
* @param model
|
* @param model
|
||||||
* @return
|
* @return
|
||||||
*/
|
*/
|
||||||
@@ -4360,7 +4467,7 @@ public class PropertyController {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* 楼栋查询接口
|
* 楼栋查询接口
|
||||||
*
|
*
|
||||||
* @param model
|
* @param model
|
||||||
* @return
|
* @return
|
||||||
*/
|
*/
|
||||||
@@ -4813,7 +4920,7 @@ public class PropertyController {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* 房产交易记录查询
|
* 房产交易记录查询
|
||||||
*
|
*
|
||||||
* @param model
|
* @param model
|
||||||
* @return
|
* @return
|
||||||
*/
|
*/
|
||||||
@@ -4873,7 +4980,7 @@ public class PropertyController {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* 房产拍卖记录
|
* 房产拍卖记录
|
||||||
*
|
*
|
||||||
* @param model
|
* @param model
|
||||||
* @return
|
* @return
|
||||||
*/
|
*/
|
||||||
@@ -4934,7 +5041,7 @@ public class PropertyController {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* 房产近期评估次数
|
* 房产近期评估次数
|
||||||
*
|
*
|
||||||
* @param model
|
* @param model
|
||||||
* @return
|
* @return
|
||||||
*/
|
*/
|
||||||
@@ -5650,11 +5757,15 @@ public class PropertyController {
|
|||||||
this.rentDealPriceService = rentDealPriceService;
|
this.rentDealPriceService = rentDealPriceService;
|
||||||
}
|
}
|
||||||
|
|
||||||
public CommunityPoiServiceImpl getCommunityPoiService() {
|
public void setCommunityPoiService(CommunityPoiService communityPoiService) {
|
||||||
return communityPoiService;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setCommunityPoiService(CommunityPoiServiceImpl communityPoiService) {
|
|
||||||
this.communityPoiService = communityPoiService;
|
this.communityPoiService = communityPoiService;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void setEvaluateFactorService(EvaluateFactorService evaluateFactorService) {
|
||||||
|
this.evaluateFactorService = evaluateFactorService;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setCommunityBasePriceService(CommunityBasePriceService communityBasePriceService) {
|
||||||
|
this.communityBasePriceService = communityBasePriceService;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-1
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
|||||||
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
||||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
||||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220318, 20220419);
|
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220418, 20220519);
|
||||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
|
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
|
||||||
//测试通过匹配接口配房源效果
|
//测试通过匹配接口配房源效果
|
||||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
|
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
|
||||||
|
|||||||
+2
-2
@@ -38,8 +38,8 @@ class ExportOracleData {
|
|||||||
exportCommunityBuileType()
|
exportCommunityBuileType()
|
||||||
|
|
||||||
exportThirdPartyPrice2(lastMonth)
|
exportThirdPartyPrice2(lastMonth)
|
||||||
exportBigdataForsale("2022-03-19")
|
exportBigdataForsale("2022-04-19")
|
||||||
exportBigdataDeal("2022-03-01")
|
exportBigdataDeal("2022-04-01")
|
||||||
|
|
||||||
|
|
||||||
//计算结果相关
|
//计算结果相关
|
||||||
|
|||||||
+67
@@ -0,0 +1,67 @@
|
|||||||
|
package com.fangjia.script.ccbevaluation
|
||||||
|
|
||||||
|
import com.fangjia.core.RegionData
|
||||||
|
import com.fangjia.core.util.DistrictCodeUtils
|
||||||
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
|
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||||
|
import com.fangjia.script.lixue.util.DBUtils
|
||||||
|
import com.fangjia.script.util.DateUtils
|
||||||
|
import com.fangjia.script.util.DbUtils
|
||||||
|
import com.fangjia.script.util.ElasticUtils
|
||||||
|
import com.fangjia.script.util.ExcelUtils
|
||||||
|
import com.fangjia.script.util.MongoUtils
|
||||||
|
import com.mongodb.BasicDBObject
|
||||||
|
import com.mongodb.DBCollection
|
||||||
|
import com.mongodb.DBObject
|
||||||
|
import groovy.sql.Sql
|
||||||
|
import org.apache.lucene.queryparser.classic.QueryParser
|
||||||
|
import org.apache.poi.ss.util.RegionUtil
|
||||||
|
import org.bson.types.ObjectId
|
||||||
|
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||||
|
import org.elasticsearch.index.query.QueryBuilder
|
||||||
|
import org.elasticsearch.index.query.QueryBuilders
|
||||||
|
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||||
|
|
||||||
|
import java.sql.ResultSet
|
||||||
|
import java.util.concurrent.ConcurrentHashMap
|
||||||
|
import java.util.concurrent.CountDownLatch
|
||||||
|
import java.util.concurrent.ExecutorService
|
||||||
|
import java.util.concurrent.Executors
|
||||||
|
import java.util.concurrent.atomic.AtomicLong
|
||||||
|
|
||||||
|
/**
|
||||||
|
* oralce中小区统计数据同步至85中
|
||||||
|
*/
|
||||||
|
class OracleDistrict2Mongo85 {
|
||||||
|
|
||||||
|
static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||||
|
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||||
|
|
||||||
|
static final DBCollection communityPoiConn = MongoUtils.getCollectionByName("fangjia_stat", "community_poi_gj", "192.168.0.85", 28017)
|
||||||
|
|
||||||
|
static main(args) {
|
||||||
|
// 只写入重庆市数据
|
||||||
|
def sql = "select cityid, distname, commid, commname, subway, traffic, bank, primary, junior, hospital, funeral from COMMUNITY_POI_GJ"
|
||||||
|
def count = 0
|
||||||
|
sqlInstance.eachRow(sql, { r ->
|
||||||
|
def city = DistrictCodeUtils.getCityName(r['cityid'])
|
||||||
|
def region = RegionData.formatRegion(city, r['distname'])
|
||||||
|
DBObject dbObject = new BasicDBObject("_id", r['commid'])
|
||||||
|
DBObject updateObject = new BasicDBObject("city", city).append("region", region)
|
||||||
|
.append("name", r['commname'])
|
||||||
|
.append('subway', Integer.parseInt(r['subway'].toString()))
|
||||||
|
.append("traffic", Integer.parseInt(r['traffic'].toString()))
|
||||||
|
.append("bank", Integer.parseInt(r['bank'].toString()))
|
||||||
|
.append("primary", Integer.parseInt(r['primary'].toString()))
|
||||||
|
.append("junior", Integer.parseInt(r['junior'].toString()))
|
||||||
|
.append("hospital", Integer.parseInt(r['hospital'].toString()))
|
||||||
|
.append("funeral", Integer.parseInt(r['funeral'].toString()))
|
||||||
|
.append("c_date", new Date());
|
||||||
|
communityPoiConn.update(dbObject, updateObject, true, false)
|
||||||
|
count++;
|
||||||
|
if (count % 500 == 0) {
|
||||||
|
println "已写入" + count + "条小区数据"
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
+3
-3
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
|
|||||||
@BatchOwner(Watcher.GaoJuan)
|
@BatchOwner(Watcher.GaoJuan)
|
||||||
class ThirdPartyPriceBatch extends BaseBatch {
|
class ThirdPartyPriceBatch extends BaseBatch {
|
||||||
|
|
||||||
def sDate= 202203//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
def sDate= 202204//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||||
def startDate = 202201
|
def startDate = 202202
|
||||||
def endDate = 202203
|
def endDate = 202204
|
||||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||||
|
|
||||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||||
|
|||||||
+8
-8
@@ -40,8 +40,8 @@ class TradeStatLog {
|
|||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new TradeStatLog()
|
def task = new TradeStatLog()
|
||||||
// task.run()//统计最近3个月抓取的成交数据量
|
task.run()//统计最近3个月抓取的成交数据量
|
||||||
task.exportTrade()//导出未入库成交
|
// task.exportTrade()//导出未入库成交
|
||||||
// task.thirdPartyStat()
|
// task.thirdPartyStat()
|
||||||
// println Calendar.getInstance().get(Calendar.YEAR)
|
// println Calendar.getInstance().get(Calendar.YEAR)
|
||||||
// task.averageStat()
|
// task.averageStat()
|
||||||
@@ -75,13 +75,13 @@ class TradeStatLog {
|
|||||||
def run(){
|
def run(){
|
||||||
def citys = CcbCommonUtils.citys;
|
def citys = CcbCommonUtils.citys;
|
||||||
def rets = []
|
def rets = []
|
||||||
rets << ["城市","1月总量","2月总量","3月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
rets << ["城市","2月总量","3月总量","4月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||||
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
|
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
|
||||||
//def startDate = "2021-05-01"
|
//def startDate = "2021-05-01"
|
||||||
def startDate = "2022-01-01"
|
def startDate = "2022-02-01"
|
||||||
def date = "2022-02-01"
|
def date = "2022-03-01"
|
||||||
def endDate = "2022-03-01"
|
def endDate = "2022-04-01"
|
||||||
def newDate = "2022-04-01"
|
def newDate = "2022-05-01"
|
||||||
|
|
||||||
for(def city : citys){
|
for(def city : citys){
|
||||||
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||||
@@ -131,7 +131,7 @@ class TradeStatLog {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-03成交统计.xlsx", rets, "数据")
|
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-04成交统计.xlsx", rets, "数据")
|
||||||
println "表格生成完成"
|
println "表格生成完成"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+117
@@ -0,0 +1,117 @@
|
|||||||
|
package com.fangjia.script.ccbevaluation.data
|
||||||
|
|
||||||
|
import com.fangjia.core.RegionData
|
||||||
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
|
import com.fangjia.script.util.ExcelUtils
|
||||||
|
import com.fangjia.script.util.OpenApiClient
|
||||||
|
import groovy.sql.Sql
|
||||||
|
|
||||||
|
class CcbDistNameMatch {
|
||||||
|
|
||||||
|
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||||
|
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||||
|
|
||||||
|
static main(args) {
|
||||||
|
def task = new CcbDistNameMatch();
|
||||||
|
// task.run()
|
||||||
|
task.run1()
|
||||||
|
}
|
||||||
|
|
||||||
|
def run1(){
|
||||||
|
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\新增映射别名.xlsx", { d ->
|
||||||
|
if (d['A']) {
|
||||||
|
def region = d['B']
|
||||||
|
def newRegion = RegionData.formatRegion(d['A'], region)
|
||||||
|
if (newRegion) {
|
||||||
|
if(d['D'].contains("、")){
|
||||||
|
println d['D']
|
||||||
|
}
|
||||||
|
def arrs = d['D'].split("、")
|
||||||
|
def list = []
|
||||||
|
def flags = []
|
||||||
|
for(def s : arrs){
|
||||||
|
def dist = CommonDatautils.findDistrict(d['A'], newRegion, s, "district")
|
||||||
|
if(dist){
|
||||||
|
list << dist.name
|
||||||
|
flags << "完全一致"
|
||||||
|
}else{
|
||||||
|
def jsonObject = OpenApiClient.addressMatch(d['A'], s, "property", false, newRegion)
|
||||||
|
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||||
|
if(jsonObject.get("district") == newRegion){
|
||||||
|
list << jsonObject.get("name")
|
||||||
|
flags << "匹配接口"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
if(list){
|
||||||
|
list = list.unique()
|
||||||
|
d['E'] = list.join(",")
|
||||||
|
d['F'] = flags.join(",")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
def run(){
|
||||||
|
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\不同小区.xlsx", { d ->
|
||||||
|
def regionCode = null
|
||||||
|
if (d['A']) {
|
||||||
|
def region = d['B']
|
||||||
|
def newRegion = RegionData.formatRegion(d['A'], region)
|
||||||
|
if (newRegion) {
|
||||||
|
def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
|
||||||
|
if(dist){
|
||||||
|
d['D'] = dist.name
|
||||||
|
def count = findDistrictByOracle(dist._id.toString())
|
||||||
|
if(count && count>0){
|
||||||
|
d['F'] = 1
|
||||||
|
}else{
|
||||||
|
d['H'] = dist.estate_type2
|
||||||
|
d['J'] = dist.name_cat
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['C'], "district")
|
||||||
|
if (rets) {
|
||||||
|
if(rets.size()>1){
|
||||||
|
d['E'] = "别名-多个"
|
||||||
|
}else{
|
||||||
|
dist = rets[0]
|
||||||
|
d['E'] = d['C'] +"--"+dist.name
|
||||||
|
def count = findAliasByOracle(dist._id.toString())
|
||||||
|
if(count && count>0) {
|
||||||
|
d['G'] = 1
|
||||||
|
}else{
|
||||||
|
d['H'] = dist.estate_type2
|
||||||
|
d['J'] = dist.name_cat
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
def findDistrictByOracle(commid){
|
||||||
|
def sql = "select count(*) as count from bigdata_community_gj where commid='${commid}' and status=0";
|
||||||
|
def count = 0
|
||||||
|
|
||||||
|
sqlInstance.eachRow(sql,{r->
|
||||||
|
count = r.count
|
||||||
|
});
|
||||||
|
return count
|
||||||
|
}
|
||||||
|
|
||||||
|
def findAliasByOracle(commid){
|
||||||
|
def sql = "select count(*) as count from address_alias_fjw where address_id='${commid}'";
|
||||||
|
def count = 0
|
||||||
|
|
||||||
|
sqlInstance.eachRow(sql,{r->
|
||||||
|
count = r.count
|
||||||
|
});
|
||||||
|
return count
|
||||||
|
}
|
||||||
|
}
|
||||||
+14
-13
@@ -23,22 +23,22 @@ class MatchEvaluateRegionTest {
|
|||||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||||
|
|
||||||
def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx"
|
def path = "D:\\数据\\重庆农商行\\2022-05-06\\估值限时测试-重庆农商行1.0.xlsx"
|
||||||
def total = 1500
|
def total = 1500
|
||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new MatchEvaluateRegionTest()
|
def task = new MatchEvaluateRegionTest()
|
||||||
task.run()//小区匹配
|
// task.run()//小区匹配
|
||||||
task.run1()//地图匹配
|
// task.run1()//地图匹配
|
||||||
// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除
|
// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除
|
||||||
task.addPrice()
|
task.addPrice()
|
||||||
task.addArroundPrice()
|
// task.addArroundPrice()
|
||||||
task.addAuthor()
|
// task.addAuthor()
|
||||||
}
|
}
|
||||||
def run(){
|
def run(){
|
||||||
ExcelUtils.multiThreadingModify(path, { row ->
|
ExcelUtils.multiThreadingModify(path, { row ->
|
||||||
if(!row['F']){
|
if(!row['F']){
|
||||||
def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], "property", false, null, null)
|
def jsonObject = OpenApiClient.addressMatch("重庆", row['C'], "property", false,null,null,)
|
||||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||||
row['F'] = jsonObject.get("id")
|
row['F'] = jsonObject.get("id")
|
||||||
row['G'] = jsonObject.get("city")
|
row['G'] = jsonObject.get("city")
|
||||||
@@ -57,7 +57,7 @@ class MatchEvaluateRegionTest {
|
|||||||
def run1(){
|
def run1(){
|
||||||
ExcelUtils.multiThreadingModify(path, { row ->
|
ExcelUtils.multiThreadingModify(path, { row ->
|
||||||
if(!row['F']){
|
if(!row['F']){
|
||||||
def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], null, true, null, null)
|
def jsonObject = OpenApiClient.addressMatch("重庆", row['C'], null, true, null, null)
|
||||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||||
row['F'] = jsonObject.get("id")
|
row['F'] = jsonObject.get("id")
|
||||||
row['G'] = jsonObject.get("city")
|
row['G'] = jsonObject.get("city")
|
||||||
@@ -82,7 +82,7 @@ class MatchEvaluateRegionTest {
|
|||||||
row['N']= (int)(priceObj.avg_price * row['E'])
|
row['N']= (int)(priceObj.avg_price * row['E'])
|
||||||
}
|
}
|
||||||
|
|
||||||
def avg_price = findAvgPrice(row['M'])
|
def avg_price = findAvgPrice(row['F'])
|
||||||
if(row['M'] && avg_price){
|
if(row['M'] && avg_price){
|
||||||
def weekendPrice = row['M'].toString().toDouble()
|
def weekendPrice = row['M'].toString().toDouble()
|
||||||
def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
|
def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
|
||||||
@@ -95,12 +95,12 @@ class MatchEvaluateRegionTest {
|
|||||||
|
|
||||||
def addArroundPrice(){
|
def addArroundPrice(){
|
||||||
ExcelUtils.multiThreadingModify(path, { row ->
|
ExcelUtils.multiThreadingModify(path, { row ->
|
||||||
if(row['A'] && !row['K']) {
|
if(!row['M']) {
|
||||||
def arroundObj = FangjiaClientUtils.arroundPrice(row['A'], row['B'])
|
def arroundObj = FangjiaClientUtils.arroundPrice(row['B'], row['D'])
|
||||||
if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
|
if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
|
||||||
def priceObj = arroundObj.result
|
def priceObj = arroundObj.result
|
||||||
row['M'] = priceObj.unitPrice
|
row['M'] = priceObj.unitPrice
|
||||||
row['N'] = (int) (priceObj.unitPrice * row['C'])
|
row['N'] = (int) (priceObj.unitPrice * row['E'])
|
||||||
row['Q'] = "周边价格"
|
row['Q'] = "周边价格"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -109,21 +109,22 @@ class MatchEvaluateRegionTest {
|
|||||||
|
|
||||||
def addAuthor(){
|
def addAuthor(){
|
||||||
def authors = ["丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"]
|
def authors = ["丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"]
|
||||||
def total = 747
|
|
||||||
def personTotal = authors.size()
|
def personTotal = authors.size()
|
||||||
def avg = Math.ceil(total/personTotal)
|
def avg = Math.ceil(total/personTotal)
|
||||||
|
|
||||||
def index = 0
|
def index = 0
|
||||||
def count = 0;
|
def count = 0;
|
||||||
ExcelUtils.modify(path, { row ->
|
ExcelUtils.modify(path, { row ->
|
||||||
|
if(row['B']){
|
||||||
if(index < avg){
|
if(index < avg){
|
||||||
row['P'] = authors[count]
|
row['N'] = authors[count]
|
||||||
}
|
}
|
||||||
index++
|
index++
|
||||||
if(index >= avg){
|
if(index >= avg){
|
||||||
count++
|
count++
|
||||||
index = 0
|
index = 0
|
||||||
}
|
}
|
||||||
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+9
-9
@@ -28,14 +28,14 @@ class MatchEvaluateTest {
|
|||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new MatchEvaluateTest()
|
def task = new MatchEvaluateTest()
|
||||||
// task.run()//小区匹配
|
task.run()//小区匹配
|
||||||
// task.run1()//地图匹配
|
task.run1()//地图匹配
|
||||||
//// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除
|
// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除
|
||||||
// task.addPrice()
|
task.addPrice()
|
||||||
// task.addArroundPrice()
|
task.addArroundPrice()
|
||||||
// task.addAuthor()
|
task.addAuthor()
|
||||||
|
|
||||||
task.stat()
|
// task.stat()
|
||||||
}
|
}
|
||||||
|
|
||||||
def addArroundPrice(){
|
def addArroundPrice(){
|
||||||
@@ -57,7 +57,7 @@ class MatchEvaluateTest {
|
|||||||
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
|
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
|
||||||
ExcelUtils.multiThreadingModify(path, { row ->
|
ExcelUtils.multiThreadingModify(path, { row ->
|
||||||
if(!row['D'] && row['B']){
|
if(!row['D'] && row['B']){
|
||||||
def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null)
|
def jsonObject = OpenApiClient.addressMatch5("重庆", row['B'], "property", false)
|
||||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||||
row['D'] = jsonObject.get("id")
|
row['D'] = jsonObject.get("id")
|
||||||
row['E'] = jsonObject.get("city")
|
row['E'] = jsonObject.get("city")
|
||||||
@@ -77,7 +77,7 @@ class MatchEvaluateTest {
|
|||||||
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
|
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
|
||||||
ExcelUtils.multiThreadingModify(path, { row ->
|
ExcelUtils.multiThreadingModify(path, { row ->
|
||||||
if(!row['D'] && row['B']){
|
if(!row['D'] && row['B']){
|
||||||
def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], null, true, null, null)
|
def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], null, true)
|
||||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||||
row['D'] = jsonObject.get("id")
|
row['D'] = jsonObject.get("id")
|
||||||
row['E'] = jsonObject.get("city")
|
row['E'] = jsonObject.get("city")
|
||||||
|
|||||||
@@ -0,0 +1,763 @@
|
|||||||
|
package com.fangjia.script.chongqing
|
||||||
|
|
||||||
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
|
import com.fangjia.script.common.DataHandleUtils
|
||||||
|
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
|
||||||
|
import com.fangjia.script.util.ExcelUtils
|
||||||
|
import com.fangjia.script.util.OpenApiClient
|
||||||
|
|
||||||
|
import java.util.concurrent.CountDownLatch
|
||||||
|
import java.util.concurrent.ExecutorService
|
||||||
|
import java.util.concurrent.Executors
|
||||||
|
import java.util.concurrent.atomic.AtomicLong
|
||||||
|
|
||||||
|
class PocStat {
|
||||||
|
|
||||||
|
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
|
||||||
|
|
||||||
|
static main(args) {
|
||||||
|
def task = new PocStat();
|
||||||
|
task.regionDistStat();
|
||||||
|
// task.sampleStat()
|
||||||
|
// task.tradeStat()
|
||||||
|
// task.paimaiStat()
|
||||||
|
// task.evalStat()
|
||||||
|
//
|
||||||
|
// task.fileldStat()
|
||||||
|
}
|
||||||
|
|
||||||
|
def regionDistStat(){
|
||||||
|
def city = "重庆"
|
||||||
|
def regionMap = CommonDatautils.getRegionsByCity(city)
|
||||||
|
def set =regionMap.keySet();
|
||||||
|
def rows = []
|
||||||
|
rows << ["区域","小区量"]
|
||||||
|
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name"])
|
||||||
|
for(def region : set){
|
||||||
|
def count = CommonDatautils.findDistrictCountByRegionBlock(city, region, null, "district")
|
||||||
|
|
||||||
|
def regionCount = 0
|
||||||
|
def regionHouseLDCount = 0
|
||||||
|
for(def d : dists){
|
||||||
|
if(d.region == region){
|
||||||
|
def loudongCount = CommonDatautils.findLoudong(d._id.toString())
|
||||||
|
if(loudongCount && loudongCount>0){
|
||||||
|
regionCount = regionCount+ loudongCount
|
||||||
|
}
|
||||||
|
|
||||||
|
def houseLouDongCount = CommonDatautils.findHouseLoudong(d._id.toString())
|
||||||
|
if(houseLouDongCount && houseLouDongCount>0){
|
||||||
|
regionHouseLDCount = regionHouseLDCount + houseLouDongCount
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rows << [region, count, regionCount, regionHouseLDCount]
|
||||||
|
println region +" 完成"
|
||||||
|
}
|
||||||
|
ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx",rows, "各区楼盘量统计")
|
||||||
|
}
|
||||||
|
|
||||||
|
def sampleStat(){
|
||||||
|
def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"])
|
||||||
|
|
||||||
|
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||||
|
|
||||||
|
def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"]
|
||||||
|
def zCityDistTotal = new AtomicLong()
|
||||||
|
def fCityDistTotal = new AtomicLong()
|
||||||
|
//[0, 1), [1, 5), [5, 20), [20, 50), [50,+)
|
||||||
|
def zHLevel1 = new AtomicLong()
|
||||||
|
def zHLevel2 = new AtomicLong()
|
||||||
|
def zHLevel3 = new AtomicLong()
|
||||||
|
def zHLevel4 = new AtomicLong()
|
||||||
|
def zHLevel5 = new AtomicLong()
|
||||||
|
|
||||||
|
def fHLevel1 = new AtomicLong()
|
||||||
|
def fHLevel2 = new AtomicLong()
|
||||||
|
def fHLevel3 = new AtomicLong()
|
||||||
|
def fHLevel4 = new AtomicLong()
|
||||||
|
def fHLevel5 = new AtomicLong()
|
||||||
|
|
||||||
|
def rets = []
|
||||||
|
rets << ["城区","小区量","[0, 1)","[1, 5)","[5, 20)","[20, 50)","[50,+)"]
|
||||||
|
|
||||||
|
|
||||||
|
def index = new AtomicLong()
|
||||||
|
for(def i=0;i<dists.size();i++) {
|
||||||
|
final def d = dists[i]
|
||||||
|
fixedThreadPool.execute(new Runnable() {
|
||||||
|
void run() {
|
||||||
|
try {
|
||||||
|
index.addAndGet(1)
|
||||||
|
// println index +" "+d.name
|
||||||
|
def commid = d._id.toString()
|
||||||
|
//房源
|
||||||
|
def jsonObject1 = OpenApiClient.houseList(commid,"20220201",60)
|
||||||
|
if(jsonObject1 && jsonObject1.result) {
|
||||||
|
def results = jsonObject1.result
|
||||||
|
|
||||||
|
if(zRegions.contains(d.region)){
|
||||||
|
zCityDistTotal.addAndGet(1)
|
||||||
|
if(results){
|
||||||
|
if(results.size()<5){
|
||||||
|
zHLevel2.addAndGet(1)
|
||||||
|
}else if(results.size()<20){
|
||||||
|
zHLevel3.addAndGet(1)
|
||||||
|
}else if(results.size()<50){
|
||||||
|
zHLevel4.addAndGet(1)
|
||||||
|
}else if(results.size()>=50){
|
||||||
|
zHLevel5.addAndGet(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
fCityDistTotal.addAndGet(1)
|
||||||
|
|
||||||
|
if(results){
|
||||||
|
if(results.size()<5){
|
||||||
|
fHLevel2.addAndGet(1)
|
||||||
|
}else if(results.size()<20){
|
||||||
|
fHLevel3.addAndGet(1)
|
||||||
|
}else if(results.size()<50){
|
||||||
|
fHLevel4.addAndGet(1)
|
||||||
|
}else if(results.size()>=50){
|
||||||
|
fHLevel5.addAndGet(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
if(zRegions.contains(d.region)) {
|
||||||
|
zCityDistTotal.addAndGet(1)
|
||||||
|
zHLevel1.addAndGet(1)
|
||||||
|
|
||||||
|
println d.city+"#"+d.region+"#"+d.name
|
||||||
|
}else{
|
||||||
|
fCityDistTotal.addAndGet(1)
|
||||||
|
fHLevel1.addAndGet(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace()
|
||||||
|
} finally {
|
||||||
|
latch.countDown()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
latch.await()
|
||||||
|
def ret1 = ["主城区", zCityDistTotal.get(), zHLevel1.get(), zHLevel2.get(), zHLevel3.get(), zHLevel4.get(), zHLevel5.get()]
|
||||||
|
def ret2 = ["非主城区", fCityDistTotal.get(), fHLevel1.get(), fHLevel2.get(), fHLevel3.get(), fHLevel4.get(), fHLevel5.get()]
|
||||||
|
rets.add(ret1)
|
||||||
|
rets.add(ret2)
|
||||||
|
ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx", rets, "案例数量分布-样本")
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def tradeStat(){
|
||||||
|
def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"])
|
||||||
|
|
||||||
|
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||||
|
|
||||||
|
def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"]
|
||||||
|
def zCityDistTotal = new AtomicLong()
|
||||||
|
def fCityDistTotal = new AtomicLong()
|
||||||
|
//[0, 1), [1, 5), [5, 20), [20, 50), [50,+)
|
||||||
|
def zTLevel1 = new AtomicLong()
|
||||||
|
def zTLevel2 = new AtomicLong()
|
||||||
|
def zTLevel3 = new AtomicLong()
|
||||||
|
def zTLevel4 = new AtomicLong()
|
||||||
|
def zTLevel5 = new AtomicLong()
|
||||||
|
|
||||||
|
def fTLevel1 = new AtomicLong()
|
||||||
|
def fTLevel2 = new AtomicLong()
|
||||||
|
def fTLevel3 = new AtomicLong()
|
||||||
|
def fTLevel4 = new AtomicLong()
|
||||||
|
def fTLevel5 = new AtomicLong()
|
||||||
|
|
||||||
|
def rets = []
|
||||||
|
rets << ["城区","小区量","[0, 1)","[1, 5)","[5, 20)","[20, 50)","[50,+)"]
|
||||||
|
|
||||||
|
|
||||||
|
def index = new AtomicLong()
|
||||||
|
for(def i=0;i<dists.size();i++) {
|
||||||
|
final def d = dists[i]
|
||||||
|
fixedThreadPool.execute(new Runnable() {
|
||||||
|
void run() {
|
||||||
|
try {
|
||||||
|
index.addAndGet(1)
|
||||||
|
println index +" "+d.name
|
||||||
|
def commid = d._id.toString()
|
||||||
|
//房源
|
||||||
|
def jsonObject1 = OpenApiClient.searchDealPrice(commid,"20220201",60)
|
||||||
|
if(jsonObject1 && jsonObject1.result) {
|
||||||
|
def results = jsonObject1.result
|
||||||
|
|
||||||
|
if(zRegions.contains(d.region)){
|
||||||
|
zCityDistTotal.addAndGet(1)
|
||||||
|
if(results){
|
||||||
|
if(results.size()<5){
|
||||||
|
zTLevel2.addAndGet(1)
|
||||||
|
}else if(results.size()<20){
|
||||||
|
zTLevel3.addAndGet(1)
|
||||||
|
}else if(results.size()<50){
|
||||||
|
zTLevel4.addAndGet(1)
|
||||||
|
}else if(results.size()>=50){
|
||||||
|
zTLevel5.addAndGet(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
fCityDistTotal.addAndGet(1)
|
||||||
|
|
||||||
|
if(results){
|
||||||
|
if(results.size()<5){
|
||||||
|
fTLevel2.addAndGet(1)
|
||||||
|
}else if(results.size()<20){
|
||||||
|
fTLevel3.addAndGet(1)
|
||||||
|
}else if(results.size()<50){
|
||||||
|
fTLevel4.addAndGet(1)
|
||||||
|
}else if(results.size()>=50){
|
||||||
|
fTLevel5.addAndGet(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
if(zRegions.contains(d.region)) {
|
||||||
|
zCityDistTotal.addAndGet(1)
|
||||||
|
zTLevel1.addAndGet(1)
|
||||||
|
}else{
|
||||||
|
fCityDistTotal.addAndGet(1)
|
||||||
|
fTLevel1.addAndGet(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace()
|
||||||
|
} finally {
|
||||||
|
latch.countDown()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
latch.await()
|
||||||
|
def ret1 = ["主城区", zCityDistTotal.get(), zTLevel1.get(), zTLevel2.get(), zTLevel3.get(), zTLevel4.get(), zTLevel5.get()]
|
||||||
|
def ret2 = ["非主城区", fCityDistTotal.get(), fTLevel1.get(), fTLevel2.get(), fTLevel3.get(), fTLevel4.get(), fTLevel5.get()]
|
||||||
|
rets.add(ret1)
|
||||||
|
rets.add(ret2)
|
||||||
|
ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx", rets, "案例数量分布-成交")
|
||||||
|
println "成交案例统计完成"
|
||||||
|
}
|
||||||
|
|
||||||
|
def paimaiStat(){
|
||||||
|
def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"])
|
||||||
|
|
||||||
|
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||||
|
|
||||||
|
def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"]
|
||||||
|
def zCityDistTotal = new AtomicLong()
|
||||||
|
def fCityDistTotal = new AtomicLong()
|
||||||
|
//[0, 1), [1, 5), [5, 20), [20, 50), [50,+)
|
||||||
|
def zPLevel1 = new AtomicLong()
|
||||||
|
def zPLevel2 = new AtomicLong()
|
||||||
|
def zPLevel3 = new AtomicLong()
|
||||||
|
def zPLevel4 = new AtomicLong()
|
||||||
|
def zPLevel5 = new AtomicLong()
|
||||||
|
|
||||||
|
def fPLevel1 = new AtomicLong()
|
||||||
|
def fPLevel2 = new AtomicLong()
|
||||||
|
def fPLevel3 = new AtomicLong()
|
||||||
|
def fPLevel4 = new AtomicLong()
|
||||||
|
def fPLevel5 = new AtomicLong()
|
||||||
|
|
||||||
|
def rets = []
|
||||||
|
rets << ["城区","小区量","[0, 1)","[1, 5)","[5, 20)","[20, 50)","[50,+)"]
|
||||||
|
|
||||||
|
|
||||||
|
def index = new AtomicLong()
|
||||||
|
for(def i=0;i<dists.size();i++) {
|
||||||
|
final def d = dists[i]
|
||||||
|
fixedThreadPool.execute(new Runnable() {
|
||||||
|
void run() {
|
||||||
|
try {
|
||||||
|
index.addAndGet(1)
|
||||||
|
println index +" "+d.name
|
||||||
|
def commid = d._id.toString()
|
||||||
|
//房源
|
||||||
|
def jsonObject1 = OpenApiClient.searchPaimais(d.city, d.region, d.name,"20220201",100)
|
||||||
|
if(jsonObject1 && jsonObject1.result) {
|
||||||
|
def results = jsonObject1.result
|
||||||
|
|
||||||
|
if(zRegions.contains(d.region)){
|
||||||
|
zCityDistTotal.addAndGet(1)
|
||||||
|
if(results){
|
||||||
|
if(results.size()<5){
|
||||||
|
zPLevel2.addAndGet(1)
|
||||||
|
}else if(results.size()<20){
|
||||||
|
zPLevel3.addAndGet(1)
|
||||||
|
}else if(results.size()<50){
|
||||||
|
zPLevel4.addAndGet(1)
|
||||||
|
}else if(results.size()>=50){
|
||||||
|
zPLevel5.addAndGet(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
fCityDistTotal.addAndGet(1)
|
||||||
|
|
||||||
|
if(results){
|
||||||
|
if(results.size()<5){
|
||||||
|
fPLevel2.addAndGet(1)
|
||||||
|
}else if(results.size()<20){
|
||||||
|
fPLevel3.addAndGet(1)
|
||||||
|
}else if(results.size()<50){
|
||||||
|
fPLevel4.addAndGet(1)
|
||||||
|
}else if(results.size()>=50){
|
||||||
|
fPLevel5.addAndGet(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
if(zRegions.contains(d.region)) {
|
||||||
|
zCityDistTotal.addAndGet(1)
|
||||||
|
zPLevel1.addAndGet(1)
|
||||||
|
}else{
|
||||||
|
fCityDistTotal.addAndGet(1)
|
||||||
|
fPLevel1.addAndGet(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace()
|
||||||
|
} finally {
|
||||||
|
latch.countDown()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
latch.await()
|
||||||
|
def ret1 = ["主城区", zCityDistTotal.get(), zPLevel1.get(), zPLevel2.get(), zPLevel3.get(), zPLevel4.get(), zPLevel5.get()]
|
||||||
|
def ret2 = ["非主城区", fCityDistTotal.get(), fPLevel1.get(), fPLevel2.get(), fPLevel3.get(), fPLevel4.get(), fPLevel5.get()]
|
||||||
|
rets.add(ret1)
|
||||||
|
rets.add(ret2)
|
||||||
|
ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx", rets, "案例数量分布-拍卖")
|
||||||
|
println "拍卖案例统计完成"
|
||||||
|
}
|
||||||
|
def evalStat(){
|
||||||
|
def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"])
|
||||||
|
|
||||||
|
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||||
|
|
||||||
|
def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"]
|
||||||
|
def zCityDistTotal = new AtomicLong()
|
||||||
|
def fCityDistTotal = new AtomicLong()
|
||||||
|
//[0, 1), [1, 5), [5, 20), [20, 50), [50,+)
|
||||||
|
def zELevel1 = new AtomicLong()
|
||||||
|
def zELevel2 = new AtomicLong()
|
||||||
|
def zELevel3 = new AtomicLong()
|
||||||
|
def zELevel4 = new AtomicLong()
|
||||||
|
def zELevel5 = new AtomicLong()
|
||||||
|
|
||||||
|
def fELevel1 = new AtomicLong()
|
||||||
|
def fELevel2 = new AtomicLong()
|
||||||
|
def fELevel3 = new AtomicLong()
|
||||||
|
def fELevel4 = new AtomicLong()
|
||||||
|
def fELevel5 = new AtomicLong()
|
||||||
|
|
||||||
|
def rets = []
|
||||||
|
rets << ["城区","小区量","[0, 1)","[1, 5)","[5, 20)","[20, 50)","[50,+)"]
|
||||||
|
|
||||||
|
|
||||||
|
def index = new AtomicLong()
|
||||||
|
for(def i=0;i<dists.size();i++) {
|
||||||
|
final def d = dists[i]
|
||||||
|
fixedThreadPool.execute(new Runnable() {
|
||||||
|
void run() {
|
||||||
|
try {
|
||||||
|
index.addAndGet(1)
|
||||||
|
println index +" "+d.name
|
||||||
|
def commid = d._id.toString()
|
||||||
|
//房源
|
||||||
|
def jsonObject1 = OpenApiClient.evaluaionList(commid,"20220201",60)
|
||||||
|
if(jsonObject1 && jsonObject1.result) {
|
||||||
|
def results = jsonObject1.result
|
||||||
|
|
||||||
|
if(zRegions.contains(d.region)){
|
||||||
|
zCityDistTotal.addAndGet(1)
|
||||||
|
if(results){
|
||||||
|
if(results.size()<5){
|
||||||
|
zELevel2.addAndGet(1)
|
||||||
|
}else if(results.size()<20){
|
||||||
|
zELevel3.addAndGet(1)
|
||||||
|
}else if(results.size()<50){
|
||||||
|
zELevel4.addAndGet(1)
|
||||||
|
}else if(results.size()>=50){
|
||||||
|
zELevel5.addAndGet(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
fCityDistTotal.addAndGet(1)
|
||||||
|
|
||||||
|
if(results){
|
||||||
|
if(results.size()<5){
|
||||||
|
fELevel2.addAndGet(1)
|
||||||
|
}else if(results.size()<20){
|
||||||
|
fELevel3.addAndGet(1)
|
||||||
|
}else if(results.size()<50){
|
||||||
|
fELevel4.addAndGet(1)
|
||||||
|
}else if(results.size()>=50){
|
||||||
|
fELevel5.addAndGet(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
if(zRegions.contains(d.region)) {
|
||||||
|
zCityDistTotal.addAndGet(1)
|
||||||
|
zELevel1.addAndGet(1)
|
||||||
|
}else{
|
||||||
|
fCityDistTotal.addAndGet(1)
|
||||||
|
fELevel1.addAndGet(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace()
|
||||||
|
} finally {
|
||||||
|
latch.countDown()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
latch.await()
|
||||||
|
def ret1 = ["主城区", zCityDistTotal.get(), zELevel1.get(), zELevel2.get(), zELevel3.get(), zELevel4.get(), zELevel5.get()]
|
||||||
|
def ret2 = ["非主城区", fCityDistTotal.get(), fELevel1.get(), fELevel2.get(), fELevel3.get(), fELevel4.get(), fELevel5.get()]
|
||||||
|
rets.add(ret1)
|
||||||
|
rets.add(ret2)
|
||||||
|
ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx", rets, "案例数量分布-评估")
|
||||||
|
println "评估案例统计完成"
|
||||||
|
}
|
||||||
|
|
||||||
|
def fileldStat(){
|
||||||
|
def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"])
|
||||||
|
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||||
|
def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"]
|
||||||
|
def alias_count = new AtomicLong()
|
||||||
|
def address_count = new AtomicLong()
|
||||||
|
def block_count = new AtomicLong()
|
||||||
|
def township_count = new AtomicLong()
|
||||||
|
def price_count = new AtomicLong()
|
||||||
|
def lng_count = new AtomicLong()
|
||||||
|
def finishBuildTime_count = new AtomicLong()
|
||||||
|
def developer_count = new AtomicLong()
|
||||||
|
def green_count = new AtomicLong()
|
||||||
|
def ratio_count = new AtomicLong()
|
||||||
|
def carsite_count = new AtomicLong()
|
||||||
|
def propertyManagement_count = new AtomicLong()
|
||||||
|
def managementFee_count = new AtomicLong()
|
||||||
|
def building_count = new AtomicLong()
|
||||||
|
def holed_count = new AtomicLong()
|
||||||
|
def car_holed_count = new AtomicLong()
|
||||||
|
def all_area_count = new AtomicLong()
|
||||||
|
def key_school_count = new AtomicLong()
|
||||||
|
def school_count = new AtomicLong()
|
||||||
|
def traffic_count = new AtomicLong()
|
||||||
|
def market_count = new AtomicLong()
|
||||||
|
def hospital_count = new AtomicLong()
|
||||||
|
def bank_count = new AtomicLong()
|
||||||
|
|
||||||
|
def f_alias_count = new AtomicLong()
|
||||||
|
def f_address_count = new AtomicLong()
|
||||||
|
def f_block_count = new AtomicLong()
|
||||||
|
def f_township_count = new AtomicLong()
|
||||||
|
def f_price_count = new AtomicLong()
|
||||||
|
def f_lng_count = new AtomicLong()
|
||||||
|
def f_finishBuildTime_count = new AtomicLong()
|
||||||
|
def f_developer_count = new AtomicLong()
|
||||||
|
def f_green_count = new AtomicLong()
|
||||||
|
def f_ratio_count = new AtomicLong()
|
||||||
|
def f_carsite_count = new AtomicLong()
|
||||||
|
def f_propertyManagement_count = new AtomicLong()
|
||||||
|
def f_managementFee_count = new AtomicLong()
|
||||||
|
def f_building_count = new AtomicLong()
|
||||||
|
def f_holed_count = new AtomicLong()
|
||||||
|
def f_car_holed_count = new AtomicLong()
|
||||||
|
def f_all_area_count = new AtomicLong()
|
||||||
|
def f_key_school_count = new AtomicLong()
|
||||||
|
def f_school_count = new AtomicLong()
|
||||||
|
def f_traffic_count = new AtomicLong()
|
||||||
|
def f_market_count = new AtomicLong()
|
||||||
|
def f_hospital_count = new AtomicLong()
|
||||||
|
def f_bank_count = new AtomicLong()
|
||||||
|
|
||||||
|
def zCityDistTotal = new AtomicLong()
|
||||||
|
def fCityDistTotal = new AtomicLong()
|
||||||
|
def index = new AtomicLong()
|
||||||
|
for(def i=0;i<dists.size();i++) {
|
||||||
|
final def dist = dists[i]
|
||||||
|
fixedThreadPool.execute(new Runnable() {
|
||||||
|
void run() {
|
||||||
|
try {
|
||||||
|
index.addAndGet(1)
|
||||||
|
println index +" "+dist.name
|
||||||
|
def jsonObject = OpenApiClient.districtDetail(dist.city, dist.region, dist.name)
|
||||||
|
if(jsonObject) {
|
||||||
|
|
||||||
|
if(zRegions.contains(jsonObject.get("district"))){
|
||||||
|
zCityDistTotal.addAndGet(1)
|
||||||
|
if(jsonObject.get("alias")){
|
||||||
|
alias_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
if(jsonObject.get("address")){
|
||||||
|
address_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
if(jsonObject.get("commercialArea")){//板块
|
||||||
|
block_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
if(jsonObject.get("township")){//街道***********
|
||||||
|
township_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
if(jsonObject.get("avgPrice")) {//基准价
|
||||||
|
price_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
if(jsonObject.get("lng")){//经纬度
|
||||||
|
lng_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
if(jsonObject.get("finishBuildTime")){//竣工时间
|
||||||
|
finishBuildTime_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
if(jsonObject.get("developer")){//开发商名称
|
||||||
|
developer_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
def green = SeaweedUtils.formatVirescencePercent(jsonObject.get("greenPercentage"))//绿化率
|
||||||
|
if(green){
|
||||||
|
green_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
def ratio = SeaweedUtils.formatCubagePercent(jsonObject.get("plotRatio"))//容积率
|
||||||
|
if(ratio){
|
||||||
|
ratio_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
def carsite = SeaweedUtils.formatCarSiteCount(jsonObject.get("parking"))//车位个数
|
||||||
|
if(carsite){
|
||||||
|
carsite_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
if(jsonObject.get("propertyManagement")){//物业公司
|
||||||
|
propertyManagement_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
if(jsonObject.get("managementFee")){//物业费
|
||||||
|
managementFee_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
if(jsonObject.get("buildingCount")){//住宅幢数
|
||||||
|
building_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
def holed = SeaweedUtils.formatHouseholdCount(jsonObject.get("totalUnitCount"))//总户数
|
||||||
|
if(holed){
|
||||||
|
holed_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
if(carsite && holed){
|
||||||
|
car_holed_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
def all_area = SeaweedUtils.formatAllArea(jsonObject.get("floorSpace"))//占地面积
|
||||||
|
if(all_area){
|
||||||
|
all_area_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
fCityDistTotal.addAndGet(1)
|
||||||
|
if(jsonObject.get("alias")){
|
||||||
|
f_alias_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
if(jsonObject.get("address")){
|
||||||
|
f_address_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
if(jsonObject.get("commercialArea")){//板块
|
||||||
|
f_block_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
if(jsonObject.get("township")){//街道***********
|
||||||
|
f_township_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
if(jsonObject.get("avgPrice")) {//基准价
|
||||||
|
f_price_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
if(jsonObject.get("lng")){//经纬度
|
||||||
|
f_lng_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
if(jsonObject.get("finishBuildTime")){//竣工时间
|
||||||
|
f_finishBuildTime_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
if(jsonObject.get("developer")){//开发商名称
|
||||||
|
f_developer_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
def green = SeaweedUtils.formatVirescencePercent(jsonObject.get("greenPercentage"))//绿化率
|
||||||
|
if(green){
|
||||||
|
f_green_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
def ratio = SeaweedUtils.formatCubagePercent(jsonObject.get("plotRatio"))//容积率
|
||||||
|
if(ratio){
|
||||||
|
f_ratio_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
def carsite = SeaweedUtils.formatCarSiteCount(jsonObject.get("parking"))//车位个数
|
||||||
|
if(carsite){
|
||||||
|
f_carsite_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
if(jsonObject.get("propertyManagement")){//物业公司
|
||||||
|
f_propertyManagement_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
if(jsonObject.get("managementFee")){//物业费
|
||||||
|
f_managementFee_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
if(jsonObject.get("buildingCount")){//住宅幢数
|
||||||
|
f_building_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
def holed = SeaweedUtils.formatHouseholdCount(jsonObject.get("totalUnitCount"))//总户数
|
||||||
|
if(holed){
|
||||||
|
f_holed_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
if(carsite && holed){
|
||||||
|
f_car_holed_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
def all_area = SeaweedUtils.formatAllArea(jsonObject.get("floorSpace"))//占地面积
|
||||||
|
if(all_area){
|
||||||
|
f_all_area_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def facilityObj = OpenApiClient.facilitys(dist.city, dist.region, dist.name)
|
||||||
|
if(facilityObj && facilityObj.result){
|
||||||
|
def result = facilityObj.result
|
||||||
|
if(zRegions.contains(dist.region)){
|
||||||
|
if(result.keySchool){
|
||||||
|
key_school_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
if(result.school){
|
||||||
|
school_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
def bus = []
|
||||||
|
if(result.bus){
|
||||||
|
bus.addAll(result.bus)
|
||||||
|
}
|
||||||
|
if(result.metro){
|
||||||
|
bus.addAll(result.metro)
|
||||||
|
}
|
||||||
|
|
||||||
|
if(bus){
|
||||||
|
traffic_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
if(result.supermarket){
|
||||||
|
market_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
if(result.hospital){
|
||||||
|
hospital_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
if(result.bank){
|
||||||
|
bank_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
if(result.keySchool){
|
||||||
|
f_key_school_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
if(result.school){
|
||||||
|
f_school_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
def bus = []
|
||||||
|
if(result.bus){
|
||||||
|
bus.addAll(result.bus)
|
||||||
|
}
|
||||||
|
if(result.metro){
|
||||||
|
bus.addAll(result.metro)
|
||||||
|
}
|
||||||
|
|
||||||
|
if(bus){
|
||||||
|
f_traffic_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
if(result.supermarket){
|
||||||
|
f_market_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
if(result.hospital){
|
||||||
|
f_hospital_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
if(result.bank){
|
||||||
|
f_bank_count.addAndGet(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace()
|
||||||
|
} finally {
|
||||||
|
latch.countDown()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
latch.await()
|
||||||
|
def rets = []
|
||||||
|
rets << ["城区","小区名称","小区id","小区量","别名","地址","板块","街道","基准价","经纬度","竣工时间","开发商名称","绿化率","容积率","车位个数","物业公司","物业费","住宅幢数","总户数","车位配比","占地面积","是否学区","周边教育","周边交通","周边商业","周边医院","周边金融机构","楼盘每月均价走势","板块每月均价走势","行政区每月均价走势","城市每月均价走势"]
|
||||||
|
|
||||||
|
def alias_pre = DataHandleUtils.getDoubleRound(2, 100-alias_count*100/zCityDistTotal)
|
||||||
|
def addr_pre = DataHandleUtils.getDoubleRound(2, 100-address_count*100/zCityDistTotal)
|
||||||
|
def block_pre = DataHandleUtils.getDoubleRound(2, 100-block_count*100/zCityDistTotal)
|
||||||
|
def town_pre = DataHandleUtils.getDoubleRound(2, 100-township_count*100/zCityDistTotal)
|
||||||
|
def price_pre = DataHandleUtils.getDoubleRound(2, 100-price_count*100/zCityDistTotal)
|
||||||
|
def lat_pre = DataHandleUtils.getDoubleRound(2, 100-lng_count*100/zCityDistTotal)
|
||||||
|
def finish_pre = DataHandleUtils.getDoubleRound(2, 100-finishBuildTime_count*100/zCityDistTotal)
|
||||||
|
def developer_pre = DataHandleUtils.getDoubleRound(2, 100-developer_count*100/zCityDistTotal)
|
||||||
|
def green_pre = DataHandleUtils.getDoubleRound(2, 100-green_count*100/zCityDistTotal)
|
||||||
|
def ratio_pre = DataHandleUtils.getDoubleRound(2, 100-ratio_count*100/zCityDistTotal)
|
||||||
|
def carsite_pre = DataHandleUtils.getDoubleRound(2, 100-carsite_count*100/zCityDistTotal)
|
||||||
|
def pm_pre = DataHandleUtils.getDoubleRound(2, 100-propertyManagement_count*100/zCityDistTotal)
|
||||||
|
def mf_pre = DataHandleUtils.getDoubleRound(2, 100-managementFee_count*100/zCityDistTotal)
|
||||||
|
def building_pre = DataHandleUtils.getDoubleRound(2, 100-building_count*100/zCityDistTotal)
|
||||||
|
def holed_pre = DataHandleUtils.getDoubleRound(2, 100-holed_count*100/zCityDistTotal)
|
||||||
|
def car_holed_pre = DataHandleUtils.getDoubleRound(2, 100-car_holed_count*100/zCityDistTotal)
|
||||||
|
def all_area_pre = DataHandleUtils.getDoubleRound(2, 100-all_area_count*100/zCityDistTotal)
|
||||||
|
def key_school_pre = DataHandleUtils.getDoubleRound(2, 100-key_school_count*100/zCityDistTotal)
|
||||||
|
def school_pre = DataHandleUtils.getDoubleRound(2, 100-school_count*100/zCityDistTotal)
|
||||||
|
def traffic_pre = DataHandleUtils.getDoubleRound(2, 100-traffic_count*100/zCityDistTotal)
|
||||||
|
def market_pre = DataHandleUtils.getDoubleRound(2, 100-market_count*100/zCityDistTotal)
|
||||||
|
def hosp_pre = DataHandleUtils.getDoubleRound(2, 100-hospital_count*100/zCityDistTotal)
|
||||||
|
def bank_pre = DataHandleUtils.getDoubleRound(2, 100-bank_count*100/zCityDistTotal)
|
||||||
|
def ret1 = ["主城区","0" ,"0",zCityDistTotal.get(), alias_pre,addr_pre,block_pre,town_pre,price_pre,lat_pre,
|
||||||
|
finish_pre,developer_pre,green_pre,ratio_pre,carsite_pre,pm_pre,mf_pre,building_pre,
|
||||||
|
holed_pre,car_holed_pre,all_area_pre,key_school_pre,school_pre,traffic_pre,market_pre,
|
||||||
|
hosp_pre,bank_pre,price_pre,0,0,0]
|
||||||
|
|
||||||
|
|
||||||
|
def f_alias_pre = DataHandleUtils.getDoubleRound(2, 100-f_alias_count*100/zCityDistTotal)
|
||||||
|
def f_addr_pre = DataHandleUtils.getDoubleRound(2, 100-f_address_count*100/zCityDistTotal)
|
||||||
|
def f_block_pre = DataHandleUtils.getDoubleRound(2, 100-f_block_count*100/zCityDistTotal)
|
||||||
|
def f_town_pre = DataHandleUtils.getDoubleRound(2, 100-f_township_count*100/zCityDistTotal)
|
||||||
|
def f_price_pre = DataHandleUtils.getDoubleRound(2, 100-f_price_count*100/zCityDistTotal)
|
||||||
|
def f_lat_pre = DataHandleUtils.getDoubleRound(2, 100-f_lng_count*100/zCityDistTotal)
|
||||||
|
def f_finish_pre = DataHandleUtils.getDoubleRound(2, 100-f_finishBuildTime_count*100/zCityDistTotal)
|
||||||
|
def f_developer_pre = DataHandleUtils.getDoubleRound(2, 100-f_developer_count*100/zCityDistTotal)
|
||||||
|
def f_green_pre = DataHandleUtils.getDoubleRound(2, 100-f_green_count*100/zCityDistTotal)
|
||||||
|
def f_ratio_pre = DataHandleUtils.getDoubleRound(2, 100-f_ratio_count*100/zCityDistTotal)
|
||||||
|
def f_carsite_pre = DataHandleUtils.getDoubleRound(2, 100-f_carsite_count*100/zCityDistTotal)
|
||||||
|
def f_pm_pre = DataHandleUtils.getDoubleRound(2, 100-f_propertyManagement_count*100/zCityDistTotal)
|
||||||
|
def f_mf_pre = DataHandleUtils.getDoubleRound(2, 100-f_managementFee_count*100/zCityDistTotal)
|
||||||
|
def f_building_pre = DataHandleUtils.getDoubleRound(2, 100-f_building_count*100/zCityDistTotal)
|
||||||
|
def f_holed_pre = DataHandleUtils.getDoubleRound(2, 100-f_holed_count*100/zCityDistTotal)
|
||||||
|
def f_car_holed_pre = DataHandleUtils.getDoubleRound(2, 100-f_car_holed_count*100/zCityDistTotal)
|
||||||
|
def f_all_area_pre = DataHandleUtils.getDoubleRound(2, 100-f_all_area_count*100/zCityDistTotal)
|
||||||
|
def f_key_school_pre = DataHandleUtils.getDoubleRound(2, 100-f_key_school_count*100/zCityDistTotal)
|
||||||
|
def f_school_pre = DataHandleUtils.getDoubleRound(2, 100-f_school_count*100/zCityDistTotal)
|
||||||
|
def f_traffic_pre = DataHandleUtils.getDoubleRound(2, 100-f_traffic_count*100/zCityDistTotal)
|
||||||
|
def f_market_pre = DataHandleUtils.getDoubleRound(2, 100-f_market_count*100/zCityDistTotal)
|
||||||
|
def f_hosp_pre = DataHandleUtils.getDoubleRound(2, 100-f_hospital_count*100/zCityDistTotal)
|
||||||
|
def f_bank_pre = DataHandleUtils.getDoubleRound(2, 100-f_bank_count*100/zCityDistTotal)
|
||||||
|
|
||||||
|
def ret2 = ["非主城区","0" ,"0",fCityDistTotal.get(), f_alias_pre,f_addr_pre,f_block_pre,f_town_pre,f_price_pre,f_lat_pre,
|
||||||
|
f_finish_pre,f_developer_pre,f_green_pre,f_ratio_pre,f_carsite_pre,f_pm_pre,f_mf_pre,f_building_pre,
|
||||||
|
f_holed_pre,f_car_holed_pre,f_all_area_pre,f_key_school_pre,f_school_pre,f_traffic_pre,f_market_pre,
|
||||||
|
f_hosp_pre,f_bank_pre,f_price_pre,0,0,0]
|
||||||
|
rets.add(ret1)
|
||||||
|
rets.add(ret2)
|
||||||
|
ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx", rets, "字段-安")
|
||||||
|
println "字段统计完成"
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
+153
-70
@@ -2,9 +2,11 @@ package com.fangjia.script.chongqing
|
|||||||
|
|
||||||
import com.fangjia.core.util.SourceNameUtil
|
import com.fangjia.core.util.SourceNameUtil
|
||||||
import com.fangjia.script.commData.CommonDatautils
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
|
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||||
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
|
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
|
||||||
import com.fangjia.script.util.ExcelUtils
|
import com.fangjia.script.util.ExcelUtils
|
||||||
import com.fangjia.script.util.OpenApiClient
|
import com.fangjia.script.util.OpenApiClient
|
||||||
|
import org.bson.types.ObjectId
|
||||||
|
|
||||||
import java.util.concurrent.CopyOnWriteArrayList
|
import java.util.concurrent.CopyOnWriteArrayList
|
||||||
|
|
||||||
@@ -15,28 +17,75 @@ class PocTest {
|
|||||||
task.run();
|
task.run();
|
||||||
}
|
}
|
||||||
|
|
||||||
def run(){
|
|
||||||
// match("D:\\数据\\重庆农商行\\poc\\test.xlsx")
|
|
||||||
// addPrice("D:\\数据\\重庆农商行\\poc\\test.xlsx")
|
|
||||||
// standard("D:\\数据\\重庆农商行\\poc\\test.xlsx")
|
|
||||||
// addLoudongNum("D:\\数据\\重庆农商行\\poc\\test.xlsx")
|
|
||||||
// addDetail("D:\\数据\\重庆农商行\\poc\\test.xlsx", "D:\\数据\\重庆农商行\\poc\\楼盘详情.xlsx")
|
|
||||||
|
|
||||||
// createLoudong("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\底层数据-楼栋-户.xlsx")
|
/**
|
||||||
// createPaimai("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\法拍案例.xlsx")
|
* A 序号
|
||||||
// createHouse("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\案例.xlsx")
|
* B 城市
|
||||||
// createDealPrice("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\案例.xlsx")
|
* C 区域
|
||||||
createEvaluation("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\案例.xlsx")
|
* D 地址
|
||||||
|
* E 面积
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
def run(){
|
||||||
|
def inputfile = "D:\\数据\\重庆农商行\\poc\\重庆农商行.xlsx"
|
||||||
|
match(inputfile)
|
||||||
|
addPrice(inputfile)
|
||||||
|
addArroundPrice(inputfile)
|
||||||
|
// matchMap(inputfile)
|
||||||
|
|
||||||
|
standard(inputfile)
|
||||||
|
addLoudongNum(inputfile)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
// addDetail(inputfile, "D:\\数据\\重庆农商行\\poc\\楼盘详情.xlsx")
|
||||||
|
//
|
||||||
|
// createLoudong(inputfile,"D:\\数据\\重庆农商行\\poc\\底层数据-楼栋-户.xlsx")
|
||||||
|
// createPaimai(inputfile,"D:\\数据\\重庆农商行\\poc\\法拍案例.xlsx")
|
||||||
|
// createHouse(inputfile,"D:\\数据\\重庆农商行\\poc\\挂牌成交评估案例.xlsx")
|
||||||
|
// createDealPrice(inputfile,"D:\\数据\\重庆农商行\\poc\\挂牌成交评估案例.xlsx")
|
||||||
|
// createEvaluation(inputfile,"D:\\数据\\重庆农商行\\poc\\挂牌成交评估案例.xlsx")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def match(path){
|
def match(path){
|
||||||
ExcelUtils.multiThreadingModify(path, { row ->
|
ExcelUtils.multiThreadingModify(path, { row ->
|
||||||
if(!row['P'] && row['C']){
|
if(!row['Q'] && row['D']){
|
||||||
def jsonObject = OpenApiClient.addressMatch(row['A'], row['C'], "property", false, row['B'], null)
|
def city = row['B']
|
||||||
|
def address = row['D']
|
||||||
|
def region = row['C']
|
||||||
|
println "city:"+city+" region:"+region+" address:"+address
|
||||||
|
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, null)
|
||||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||||
row['P'] = jsonObject.get("name")
|
row['Q'] = jsonObject.get("name")
|
||||||
row['Q'] = jsonObject.get("id")
|
row['R'] = jsonObject.get("id")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
def matchMap(path){
|
||||||
|
ExcelUtils.multiThreadingModify(path, { row ->
|
||||||
|
if(!row['Q'] && row['D']){
|
||||||
|
def city = row['B']
|
||||||
|
def address = row['D']
|
||||||
|
def region = row['C']
|
||||||
|
println "city:"+city+" region:"+region+" address:"+address
|
||||||
|
def jsonObject = OpenApiClient.addressMatch(city, address, null, true, region, null)
|
||||||
|
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||||
|
def propertyType = jsonObject.get("propertyType")
|
||||||
|
if(propertyType && (propertyType.contains("住宅") || propertyType.contains("别墅") || propertyType.contains("商住"))){
|
||||||
|
row['Q'] = jsonObject.get("name")
|
||||||
|
row['R'] = new ObjectId().toString()
|
||||||
|
row['W'] = propertyType
|
||||||
|
|
||||||
|
def arroundObj = FangjiaClientUtils.arroundPrice(city, address)
|
||||||
|
if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
|
||||||
|
def priceObj = arroundObj.result
|
||||||
|
row['F'] = priceObj.unitPrice
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
@@ -44,34 +93,51 @@ class PocTest {
|
|||||||
|
|
||||||
def addPrice(path){
|
def addPrice(path){
|
||||||
ExcelUtils.multiThreadingModify(path, { row ->
|
ExcelUtils.multiThreadingModify(path, { row ->
|
||||||
if(row['Q']){
|
def commid = row['R']
|
||||||
def dist = CommonDatautils.findDistrictById(row['Q'])
|
if(commid){
|
||||||
|
def dist = CommonDatautils.findDistrictById(commid)
|
||||||
if(dist){
|
if(dist){
|
||||||
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
|
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
|
||||||
if(priceObj){
|
if(priceObj){
|
||||||
row['E'] = priceObj.avg_price
|
row['F'] = priceObj.avg_price
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def addArroundPrice(path){
|
||||||
|
ExcelUtils.multiThreadingModify(path, { row ->
|
||||||
|
if(!row['F']) {
|
||||||
|
def arroundObj = FangjiaClientUtils.arroundPrice(row['B'], row['D'])
|
||||||
|
if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
|
||||||
|
def priceObj = arroundObj.result
|
||||||
|
row['F'] = priceObj.unitPrice
|
||||||
|
row['W'] = "周边均价"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
def standard(path){
|
def standard(path){
|
||||||
ExcelUtils.multiThreadingModify(path, { row ->
|
ExcelUtils.multiThreadingModify(path, { row ->
|
||||||
if(row['C']){
|
def city = row['B']
|
||||||
def jsonObject = OpenApiClient.standard2Action(row['A'], row['C'])
|
def address = row['D']
|
||||||
|
if(address){
|
||||||
|
println "[standard] address:"+address
|
||||||
|
def jsonObject = OpenApiClient.standard2Action(city, address)
|
||||||
if(jsonObject){
|
if(jsonObject){
|
||||||
println jsonObject
|
println jsonObject
|
||||||
row['F'] = jsonObject.get("province")
|
row['G'] = jsonObject.get("province")
|
||||||
row['G'] = jsonObject.get("city")
|
row['H'] = jsonObject.get("city")
|
||||||
row['H'] = jsonObject.get("district")
|
row['I'] = jsonObject.get("district")
|
||||||
row['I'] = jsonObject.get("township")
|
row['J'] = jsonObject.get("township")
|
||||||
row['J'] = jsonObject.get("street")
|
row['K'] = jsonObject.get("street")
|
||||||
row['K'] = jsonObject.get("streetNumber")
|
row['L'] = jsonObject.get("streetNumber")
|
||||||
row['L'] = jsonObject.get("buildingNumber")
|
row['M'] = jsonObject.get("buildingNumber")
|
||||||
row['M'] = jsonObject.get("unitNumber")
|
row['N'] = jsonObject.get("unitNumber")
|
||||||
row['N'] = jsonObject.get("floor")
|
row['O'] = jsonObject.get("floor")
|
||||||
row['O'] = jsonObject.get("roomNumber")
|
row['P'] = jsonObject.get("roomNumber")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -81,23 +147,27 @@ class PocTest {
|
|||||||
def addLoudongNum(path){
|
def addLoudongNum(path){
|
||||||
|
|
||||||
ExcelUtils.multiThreadingModify(path, { row ->
|
ExcelUtils.multiThreadingModify(path, { row ->
|
||||||
if (row['Q'] && row['L']) {
|
def commid = row['R']
|
||||||
def jsonObject = OpenApiClient.searchLoudong(row['Q'], row['L'], row['M'])
|
def houseNum = row['M']
|
||||||
|
def unitNum = row['N']
|
||||||
|
println "[addLoudongNum] commid:"+commid+" houseNum:"+houseNum
|
||||||
|
if (commid && houseNum) {
|
||||||
|
def jsonObject = OpenApiClient.searchLoudong(commid, houseNum, unitNum)
|
||||||
if (jsonObject && jsonObject.result) {
|
if (jsonObject && jsonObject.result) {
|
||||||
if(jsonObject.result.size()>=1){
|
if(jsonObject.result.size()>=1){
|
||||||
def obj = jsonObject.result[0]
|
def obj = jsonObject.result[0]
|
||||||
row['S'] = obj.id
|
row['T'] = obj.id
|
||||||
row['R'] = obj.houseNumber
|
row['S'] = obj.houseNumber
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if(row['O']){
|
if(row['P']){
|
||||||
def jsonObject1 = OpenApiClient.searchHouseLoudong(row['Q'], row['L'], row['M'], row['O'])
|
def jsonObject1 = OpenApiClient.searchHouseLoudong(commid, houseNum, unitNum, row['P'])
|
||||||
if (jsonObject1 && jsonObject1.result) {
|
if (jsonObject1 && jsonObject1.result) {
|
||||||
if(jsonObject.result.size()>=1){
|
if(jsonObject1.result.size()>=1){
|
||||||
def obj = jsonObject.result[0]
|
def obj = jsonObject1.result[0]
|
||||||
row['U'] = obj.id
|
row['V'] = obj.id
|
||||||
row['T'] = obj.roomNum
|
row['U'] = obj.roomNum
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -111,12 +181,12 @@ class PocTest {
|
|||||||
rets << ["输入城市","输入区域","输入地址","面积","楼栋号","楼栋ID","物业类型","主朝向","地上层数","地下层数","竣工日期","梯","户","总户数","景观","户号","户ID","主朝向","景观","户型结构"]
|
rets << ["输入城市","输入区域","输入地址","面积","楼栋号","楼栋ID","物业类型","主朝向","地上层数","地下层数","竣工日期","梯","户","总户数","景观","户号","户ID","主朝向","景观","户型结构"]
|
||||||
for(def row : datas){
|
for(def row : datas){
|
||||||
def data = []
|
def data = []
|
||||||
data << row['A']
|
|
||||||
data << row['B']
|
data << row['B']
|
||||||
data << row['C']
|
data << row['C']
|
||||||
data << row['D']
|
data << row['D']
|
||||||
if (row['Q'] && row['L']) {
|
data << row['E']
|
||||||
def jsonObject = OpenApiClient.searchLoudong(row['Q'], row['L'], row['M'])
|
if (row['R'] && row['M']) {
|
||||||
|
def jsonObject = OpenApiClient.searchLoudong(row['R'], row['M'], row['N'])
|
||||||
if (jsonObject && jsonObject.result && jsonObject.result.size()>=1) {
|
if (jsonObject && jsonObject.result && jsonObject.result.size()>=1) {
|
||||||
def obj = jsonObject.result[0]
|
def obj = jsonObject.result[0]
|
||||||
data << obj.houseNumber
|
data << obj.houseNumber
|
||||||
@@ -144,8 +214,8 @@ class PocTest {
|
|||||||
data << ""//景观
|
data << ""//景观
|
||||||
}
|
}
|
||||||
|
|
||||||
if(row['O']){
|
if(row['P']){
|
||||||
def jsonObject1 = OpenApiClient.searchHouseLoudong(row['Q'], row['L'], row['M'], row['O'])
|
def jsonObject1 = OpenApiClient.searchHouseLoudong(row['R'], row['M'], row['N'], row['P'])
|
||||||
if (jsonObject1 && jsonObject1.result) {
|
if (jsonObject1 && jsonObject1.result) {
|
||||||
if(jsonObject1.result.size()>=1){
|
if(jsonObject1.result.size()>=1){
|
||||||
def obj = jsonObject1.result[0]
|
def obj = jsonObject1.result[0]
|
||||||
@@ -174,18 +244,20 @@ class PocTest {
|
|||||||
def rows = []
|
def rows = []
|
||||||
rows << ["输入城市","输入区域","输入地址","小区名称","小区ID","小区别名","小区地址","板块","街道","基准价","经纬度","竣工时间","开发商名称","绿化率","容积率","车位个数","物业公司","物业费","住宅幢数","总户数","车位配比","占地面积","是否学区","周边教育","周边交通","周边商业","周边医院","周边金融机构"]
|
rows << ["输入城市","输入区域","输入地址","小区名称","小区ID","小区别名","小区地址","板块","街道","基准价","经纬度","竣工时间","开发商名称","绿化率","容积率","车位个数","物业公司","物业费","住宅幢数","总户数","车位配比","占地面积","是否学区","周边教育","周边交通","周边商业","周边医院","周边金融机构"]
|
||||||
for(def d : datas){
|
for(def d : datas){
|
||||||
if(d['Q']){
|
def commid = d['R']
|
||||||
def dist = CommonDatautils.findDistrictById(d['Q'])
|
println "[addDetail] commid:"+commid
|
||||||
|
if(commid){
|
||||||
|
def dist = CommonDatautils.findDistrictById(commid)
|
||||||
if(dist){
|
if(dist){
|
||||||
def row = []
|
def row = []
|
||||||
def jsonObject = OpenApiClient.districtDetail(dist.city, dist.region, dist.name)
|
def jsonObject = OpenApiClient.districtDetail(dist.city, dist.region, dist.name)
|
||||||
if(jsonObject) {
|
if(jsonObject) {
|
||||||
|
|
||||||
row << d['A']
|
|
||||||
row << d['B']
|
row << d['B']
|
||||||
row << d['C']
|
row << d['C']
|
||||||
|
row << d['D']
|
||||||
row << jsonObject.get("name")//小区名称
|
row << jsonObject.get("name")//小区名称
|
||||||
row << d['Q']//小区ID
|
row << commid//小区ID
|
||||||
if(jsonObject.get("alias")){
|
if(jsonObject.get("alias")){
|
||||||
row << jsonObject.get("alias").join(",").replace('"',"")//小区别名
|
row << jsonObject.get("alias").join(",").replace('"',"")//小区别名
|
||||||
}else{
|
}else{
|
||||||
@@ -207,7 +279,7 @@ class PocTest {
|
|||||||
row << jsonObject.get("buildingCount")//住宅幢数
|
row << jsonObject.get("buildingCount")//住宅幢数
|
||||||
row << SeaweedUtils.formatHouseholdCount(jsonObject.get("totalUnitCount"))//总户数
|
row << SeaweedUtils.formatHouseholdCount(jsonObject.get("totalUnitCount"))//总户数
|
||||||
row << SeaweedUtils.calcCarAndHouseholdRadio(jsonObject.get("parking"), jsonObject.get("totalUnitCount"))//车位配比*********
|
row << SeaweedUtils.calcCarAndHouseholdRadio(jsonObject.get("parking"), jsonObject.get("totalUnitCount"))//车位配比*********
|
||||||
row << SeaweedUtils.formatAllArea(jsonObject.get("totalUnitCount"))//占地面积
|
row << SeaweedUtils.formatAllArea(jsonObject.get("floorSpace"))//占地面积
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -271,28 +343,35 @@ class PocTest {
|
|||||||
rows << ["输入城市","输入区域","输入地址","小区名称","标的名称","楼栋","户号","面积","案例时间","类型","状态","成交单价","评估单价"]
|
rows << ["输入城市","输入区域","输入地址","小区名称","标的名称","楼栋","户号","面积","案例时间","类型","状态","成交单价","评估单价"]
|
||||||
def keys = []
|
def keys = []
|
||||||
for(def d : datas){
|
for(def d : datas){
|
||||||
if(d['Q']) {
|
def commid = d['R']
|
||||||
def dist = CommonDatautils.findDistrictById(d['Q'])
|
println "[createPaimai] commid:"+commid
|
||||||
|
if(commid) {
|
||||||
|
def dist = CommonDatautils.findDistrictById(commid)
|
||||||
if (dist) {
|
if (dist) {
|
||||||
def key = dist.city+"#"+dist.region+"#"+dist.name
|
def key = dist.city+"#"+dist.region+"#"+dist.name
|
||||||
if(keys.contains(key)){
|
if(keys.contains(key)){
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
keys << key
|
keys << key
|
||||||
def jsonObject = OpenApiClient.searchPaimais(dist.city, dist.region, dist.name,10)
|
def jsonObject = OpenApiClient.searchPaimais(dist.city, dist.region, dist.name,"20220217",100)
|
||||||
if(jsonObject && jsonObject.result){
|
if(jsonObject && jsonObject.result){
|
||||||
def results = jsonObject.result
|
def results = jsonObject.result
|
||||||
for(def r: results){
|
for(def r: results){
|
||||||
def row = []
|
def row = []
|
||||||
row << d['A']
|
|
||||||
row << d['B']
|
row << d['B']
|
||||||
row << d['C']
|
row << d['C']
|
||||||
|
row << d['D']
|
||||||
row << r.matchName
|
row << r.matchName
|
||||||
row << r.address
|
row << r.address
|
||||||
row << r.houseNum
|
row << r.houseNum
|
||||||
row << r.roomNum
|
row << r.roomNum
|
||||||
row << r.area
|
row << r.area
|
||||||
row << r.startDate
|
if(r.startDate){
|
||||||
|
row << r.startDate.substring(0, r.startDate.indexOf("T"))
|
||||||
|
}else{
|
||||||
|
row << ""
|
||||||
|
}
|
||||||
|
|
||||||
row << r.count
|
row << r.count
|
||||||
row << r.status
|
row << r.status
|
||||||
row << r.tradeAvgPrice
|
row << r.tradeAvgPrice
|
||||||
@@ -314,13 +393,15 @@ class PocTest {
|
|||||||
rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
|
rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
|
||||||
def keys = []
|
def keys = []
|
||||||
for(def d : datas) {
|
for(def d : datas) {
|
||||||
if (d['Q']) {
|
def commid = d['R']
|
||||||
|
// println "[createHouse] commid:"+commid
|
||||||
|
if (commid) {
|
||||||
def key = d['Q']
|
def key = d['Q']
|
||||||
if (keys.contains(key)) {
|
if (keys.contains(key)) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
keys << key
|
keys << key
|
||||||
def jsonObject = OpenApiClient.houseList(d['Q'],"20210501",20)
|
def jsonObject = OpenApiClient.houseList(commid,"20210501",20)
|
||||||
if(jsonObject && jsonObject.result) {
|
if(jsonObject && jsonObject.result) {
|
||||||
def results = jsonObject.result
|
def results = jsonObject.result
|
||||||
for (def r : results) {
|
for (def r : results) {
|
||||||
@@ -368,7 +449,7 @@ class PocTest {
|
|||||||
rows << row
|
rows << row
|
||||||
}
|
}
|
||||||
|
|
||||||
println d['Q'] +" "+results.size()
|
println d['R'] +" "+results.size()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -381,13 +462,14 @@ class PocTest {
|
|||||||
rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
|
rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
|
||||||
def keys = []
|
def keys = []
|
||||||
for(def d : datas) {
|
for(def d : datas) {
|
||||||
if (d['Q']) {
|
def commid= d['R']
|
||||||
def key = d['Q']
|
println "[createEvaluation] commid:"+commid
|
||||||
if (keys.contains(key)) {
|
if (commid) {
|
||||||
|
if (keys.contains(commid)) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
keys << key
|
keys << commid
|
||||||
def jsonObject = OpenApiClient.evaluaionList(d['Q'],"20210501",20)
|
def jsonObject = OpenApiClient.evaluaionList(commid,"20210501",20)
|
||||||
if(jsonObject && jsonObject.result) {
|
if(jsonObject && jsonObject.result) {
|
||||||
def results = jsonObject.result
|
def results = jsonObject.result
|
||||||
for (def r : results) {
|
for (def r : results) {
|
||||||
@@ -432,7 +514,7 @@ class PocTest {
|
|||||||
rows << row
|
rows << row
|
||||||
}
|
}
|
||||||
|
|
||||||
println d['Q'] +" "+results.size()
|
println d['R'] +" "+results.size()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -446,13 +528,14 @@ class PocTest {
|
|||||||
rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
|
rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
|
||||||
def keys = []
|
def keys = []
|
||||||
for(def d : datas) {
|
for(def d : datas) {
|
||||||
if (d['Q']) {
|
def commid= d['R']
|
||||||
def key = d['Q']
|
println "[createDealPrice] commid:"+commid
|
||||||
if (keys.contains(key)) {
|
if (commid) {
|
||||||
|
if (keys.contains(commid)) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
keys << key
|
keys << commid
|
||||||
def jsonObject = OpenApiClient.searchDealPrice(d['Q'],"20210501",20)
|
def jsonObject = OpenApiClient.searchDealPrice(commid,"20210501",20)
|
||||||
if(jsonObject && jsonObject.result) {
|
if(jsonObject && jsonObject.result) {
|
||||||
def results = jsonObject.result
|
def results = jsonObject.result
|
||||||
for (def r : results) {
|
for (def r : results) {
|
||||||
@@ -501,7 +584,7 @@ class PocTest {
|
|||||||
rows << row
|
rows << row
|
||||||
}
|
}
|
||||||
|
|
||||||
println d['Q'] +" "+results.size()
|
println d['R'] +" "+results.size()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+62
-10
@@ -68,6 +68,10 @@ class CommonDatautils {
|
|||||||
|
|
||||||
public static final DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName2("fangjia", "sqmetre_rent", mongoDbUrl)
|
public static final DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName2("fangjia", "sqmetre_rent", mongoDbUrl)
|
||||||
|
|
||||||
|
public static final DBCollection LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia_base", "loudong", mongoDbUrl)
|
||||||
|
public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", mongoDbUrl)
|
||||||
|
|
||||||
|
|
||||||
public static DSCollection getHouseDataStoreConnection(){
|
public static DSCollection getHouseDataStoreConnection(){
|
||||||
if(house_dataStoreConnection != null) {
|
if(house_dataStoreConnection != null) {
|
||||||
return house_dataStoreConnection
|
return house_dataStoreConnection
|
||||||
@@ -586,8 +590,8 @@ class CommonDatautils {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public static def findDistrict(String city, String region, String name, String cat){
|
public static def findDistrictCount(String city, String region, String name, String cat){
|
||||||
|
|
||||||
DBObject q = new BasicDBObject();
|
DBObject q = new BasicDBObject();
|
||||||
q.put("city", city);
|
q.put("city", city);
|
||||||
if(region){
|
if(region){
|
||||||
@@ -598,7 +602,24 @@ class CommonDatautils {
|
|||||||
q.put("cat", cat)
|
q.put("cat", cat)
|
||||||
q.put("status", 0)
|
q.put("status", 0)
|
||||||
|
|
||||||
|
def count = getDataStoreConnection().count(q);
|
||||||
|
return count;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public static def findDistrict(String city, String region, String name, String cat){
|
||||||
|
|
||||||
|
DBObject q = new BasicDBObject();
|
||||||
|
q.put("city", city);
|
||||||
|
if(region){
|
||||||
|
q.put("region", region);
|
||||||
|
}
|
||||||
|
|
||||||
|
q.put("name", name);
|
||||||
|
q.put("cat", cat)
|
||||||
|
q.put("status", 0)
|
||||||
|
|
||||||
|
|
||||||
DSCursor cursor = getDataStoreConnection().find(q);
|
DSCursor cursor = getDataStoreConnection().find(q);
|
||||||
if(cursor){
|
if(cursor){
|
||||||
for(def obj : cursor){
|
for(def obj : cursor){
|
||||||
@@ -609,6 +630,7 @@ class CommonDatautils {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
public static def findDistrictList(String city, String region, String name, String cat){
|
public static def findDistrictList(String city, String region, String name, String cat){
|
||||||
|
|
||||||
DBObject q = new BasicDBObject();
|
DBObject q = new BasicDBObject();
|
||||||
@@ -704,7 +726,7 @@ class CommonDatautils {
|
|||||||
q.put("visible",0)
|
q.put("visible",0)
|
||||||
q.put("cat", cat);
|
q.put("cat", cat);
|
||||||
q.put("city", city);
|
q.put("city", city);
|
||||||
// q.put("name","吴中路9弄小区")
|
// q.put("name","ÉÍû·¼²ÝÂ̰¶")
|
||||||
|
|
||||||
DBObject keys = new BasicDBObject();
|
DBObject keys = new BasicDBObject();
|
||||||
if(fields){
|
if(fields){
|
||||||
@@ -827,16 +849,24 @@ class CommonDatautils {
|
|||||||
keys.put("out_pics", 1);
|
keys.put("out_pics", 1);
|
||||||
keys.put("lat", 1);
|
keys.put("lat", 1);
|
||||||
keys.put("lng", 1);
|
keys.put("lng", 1);
|
||||||
|
keys.put("lat2", 1);
|
||||||
|
keys.put("lng2", 1);
|
||||||
keys.put("name_cat", 1);
|
keys.put("name_cat", 1);
|
||||||
keys.put("finish_building_date", 1)
|
keys.put("finish_building_date", 1)
|
||||||
keys.put("mapping_alias", 1)
|
keys.put("mapping_alias", 1)
|
||||||
keys.put("sid", 1)
|
keys.put("sid", 1)
|
||||||
DSCursor cursor = getDataStoreConnection().find(q, keys);
|
try{
|
||||||
if(cursor){
|
DSCursor cursor = getDataStoreConnection().find(q, keys);
|
||||||
for(def obj : cursor){
|
if(cursor){
|
||||||
return obj;
|
for(def obj : cursor){
|
||||||
|
return obj;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
}catch(Exception e){
|
||||||
|
println "error_id:"+id
|
||||||
|
e.printStackTrace()
|
||||||
}
|
}
|
||||||
|
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1053,9 +1083,16 @@ class CommonDatautils {
|
|||||||
|
|
||||||
|
|
||||||
public static def queryDistrictById(def id){
|
public static def queryDistrictById(def id){
|
||||||
|
def dbo = null
|
||||||
DBObject q = new BasicDBObject();
|
DBObject q = new BasicDBObject();
|
||||||
q.put("_id", new ObjectId(id));
|
try{
|
||||||
def dbo = getDataStoreConnection().findOne(q);
|
q.put("_id", new ObjectId(id.trim().toString()));
|
||||||
|
dbo = getDataStoreConnection().findOne(q);
|
||||||
|
}catch(Exception e){
|
||||||
|
println id
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
return dbo;
|
return dbo;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2385,6 +2422,7 @@ class CommonDatautils {
|
|||||||
q.put("block", block);
|
q.put("block", block);
|
||||||
}
|
}
|
||||||
q.put("cat", cat);
|
q.put("cat", cat);
|
||||||
|
q.put("status",0)
|
||||||
def cur = getDataStoreConnection().find(q);
|
def cur = getDataStoreConnection().find(q);
|
||||||
return cur.count();
|
return cur.count();
|
||||||
}
|
}
|
||||||
@@ -3184,5 +3222,19 @@ class CommonDatautils {
|
|||||||
def avg_radios = (int)(radios.sum()/radios.size())
|
def avg_radios = (int)(radios.sum()/radios.size())
|
||||||
return avg_radios;
|
return avg_radios;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static def findLoudong(commid){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("id_in_seaweed", commid)
|
||||||
|
def count = LOUDONG_LINE_COLL.count(q)
|
||||||
|
return count;
|
||||||
|
}
|
||||||
|
|
||||||
|
static def findHouseLoudong(commid){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("district_id", commid)
|
||||||
|
def count = HOUSE_LOUDONG_COLL.count(q)
|
||||||
|
return count;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
+12
-8
@@ -35,7 +35,8 @@ class EsHousePriceStat extends BaseBatch{
|
|||||||
def run(){
|
def run(){
|
||||||
def cityMap = CommonDatautils.getAllCityRegions()
|
def cityMap = CommonDatautils.getAllCityRegions()
|
||||||
cityMap.each{city,regions->
|
cityMap.each{city,regions->
|
||||||
findPriceByCity(city, regions);
|
|
||||||
|
findPriceByCity(city, regions);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -64,9 +65,9 @@ class EsHousePriceStat extends BaseBatch{
|
|||||||
q.put("region" , r.region)
|
q.put("region" , r.region)
|
||||||
q.put("month", r.month)
|
q.put("month", r.month)
|
||||||
def mom = null
|
def mom = null
|
||||||
if(lastObj){
|
if(lastObj && lastObj.avg_price && r.price){
|
||||||
mom = (r.price -lastObj.avg_price)/lastObj.avg_price
|
mom = (r.price -lastObj.avg_price)/lastObj.avg_price
|
||||||
mom = DataHandleUtils.getDoubleRound(4, mom)
|
mom = DataHandleUtils.getDoubleRound(4, mom)
|
||||||
}
|
}
|
||||||
if(obj){
|
if(obj){
|
||||||
if(obj.avg_price != r.price){
|
if(obj.avg_price != r.price){
|
||||||
@@ -103,7 +104,7 @@ class EsHousePriceStat extends BaseBatch{
|
|||||||
|
|
||||||
def findEsAvgPrice(city, region){
|
def findEsAvgPrice(city, region){
|
||||||
def detail = [:]
|
def detail = [:]
|
||||||
detail.put("date_histogram",[ "field": "publish_date","interval": "1M","time_zone": "Asia/Shanghai","min_doc_count": 1])
|
detail.put("date_histogram",[ "field": "publish_date","interval": "1M","time_zone": "Asia/Shanghai","min_doc_count": 0])
|
||||||
detail.put("aggs",[ "1": ["moving_avg": ["buckets_path": "1-metric"]],"1-metric": ["avg": ["field": "avg_price"]]])
|
detail.put("aggs",[ "1": ["moving_avg": ["buckets_path": "1-metric"]],"1-metric": ["avg": ["field": "avg_price"]]])
|
||||||
|
|
||||||
def must = []
|
def must = []
|
||||||
@@ -143,15 +144,18 @@ class EsHousePriceStat extends BaseBatch{
|
|||||||
if(!priceValue){
|
if(!priceValue){
|
||||||
priceValue = row['1-metric']
|
priceValue = row['1-metric']
|
||||||
}
|
}
|
||||||
if(priceValue){
|
if(priceValue && priceValue.value){
|
||||||
price = priceValue.value.toInteger()
|
price = priceValue.value.toInteger()
|
||||||
}
|
}
|
||||||
result.price = price;
|
result.price = price;
|
||||||
def date = row.key_as_string.replace("-","").substring(0, 6)
|
def date = row.key_as_string.replace("-","").substring(0, 6)
|
||||||
def curMonth = DateUtils.formatDateByPattern(new Date(),"yyyyMM")
|
def curMonth = DateUtils.formatDateByPattern(new Date(),"yyyyMM")
|
||||||
if(date!=curMonth){
|
if(date!=curMonth){
|
||||||
result.month = date.toInteger()
|
if(date.toInteger()>200001 && result.price){
|
||||||
rets << result
|
result.month = date.toInteger()
|
||||||
|
rets << result
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+198
@@ -0,0 +1,198 @@
|
|||||||
|
package com.fangjia.script.gj.crawl
|
||||||
|
|
||||||
|
import com.fangjia.script.crawl.JQueryUtils
|
||||||
|
import com.fangjia.script.util.ABuYunUtil
|
||||||
|
import com.fangjia.script.util.MongoUtils
|
||||||
|
import com.mongodb.BasicDBObject
|
||||||
|
import com.mongodb.DBCollection
|
||||||
|
import net.sf.json.JSONObject
|
||||||
|
import org.jsoup.Jsoup
|
||||||
|
import org.openqa.selenium.WebDriver
|
||||||
|
import org.openqa.selenium.chrome.ChromeDriver
|
||||||
|
|
||||||
|
class AnjukeAppCrawl {
|
||||||
|
|
||||||
|
DBCollection detail = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "app_anjuke_detail")
|
||||||
|
|
||||||
|
static main(args) {
|
||||||
|
def task = new AnjukeAppCrawl();
|
||||||
|
task.run();
|
||||||
|
// task.run2()
|
||||||
|
}
|
||||||
|
|
||||||
|
def run2(){
|
||||||
|
def url = "https://m.anjuke.com/fz/community/"
|
||||||
|
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||||
|
System.setProperty("http.nonProxyHosts", "localhost");
|
||||||
|
WebDriver driver = new ChromeDriver()
|
||||||
|
def links = getRegion(url, driver)
|
||||||
|
for(def link : links){
|
||||||
|
def doc = getDocument(url, driver)
|
||||||
|
def str = doc.toString().substring(doc.toString().indexOf("<script> window.__NUXT__"), doc.toString().indexOf("</script>"))
|
||||||
|
if(str){
|
||||||
|
str = str.substring(str.indexOf("} ("), str.indexOf("));"))
|
||||||
|
def arrs = str.split(",")
|
||||||
|
pritln arrs.size()
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def getDocument(url, driver){
|
||||||
|
def stop = false
|
||||||
|
def error = 0
|
||||||
|
def doc = null
|
||||||
|
while(!stop){
|
||||||
|
try{
|
||||||
|
driver.get(url)
|
||||||
|
doc = Jsoup.parse(driver.getPageSource());
|
||||||
|
if(doc && !doc.toString().contains("访问验证-安居客") && !doc.toString().contains("502 Bad Gateway") && !doc.toString().contains("<title>登录</title>")){
|
||||||
|
stop = true;
|
||||||
|
}else{
|
||||||
|
error++
|
||||||
|
driver.manage().deleteAllCookies()
|
||||||
|
println "[${error}] error: "+url
|
||||||
|
Thread.sleep(5000)
|
||||||
|
}
|
||||||
|
}catch(Exception e){
|
||||||
|
error++
|
||||||
|
}
|
||||||
|
if(error>=20){
|
||||||
|
stop = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return doc
|
||||||
|
}
|
||||||
|
|
||||||
|
def getRegion(url, driver){
|
||||||
|
//def doc = getDocument(url, driver)
|
||||||
|
def header = [:]
|
||||||
|
header.put("accept","application/json, text/plain, */*")
|
||||||
|
header.put("accept-encoding","gzip, deflate, br")
|
||||||
|
header.put("accept-language","zh-CN,zh;q=0.9")
|
||||||
|
header.put("cookie","aQQ_ajkguid=E9796514-38C0-44D2-8CA3-1C58C3928B29; sessid=765EEB9D-E7A2-4B2C-8151-CA9CD939AB02; id58=CpQB22J8sqMA9stuJfeAAg==; twe=2; 58tj_uuid=bf6eaff8-851c-471a-a914-853242878ddc; als=0; seo_source_type=1; _ga=GA1.2.215255179.1652344695; isp=true; ajk-appVersion=; obtain_by=2; fzq_js_anjuke_ershoufang_m=c2c12b2d1b221734fa20ef752cd6bdc0_1652509223606_27; fzq_h=ef0186eeaba082ecd2a47ede7c7e3e64_1652533612190_a34c026e9c1842a3b35bdd37f3d2aa0b_2015773878; init_refer=https%253A%252F%252Fm.anjuke.com%252Fcq%252Fcommunity%252F; new_uv=8; wmda_uuid=b59ce2efbba070ecfa8fe93d79f30dd4; wmda_new_uuid=1; wmda_session_id_6145577459763=1652535525740-dae51249-3379-d4e7; wmda_visited_projects=%3B6145577459763; new_session=0; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; xxzl_deviceid=q9XQbdA9hhkHJTj+RgVpwc7jrBzVJp9J1uTlp4hDtRiXtA615K4HS4R0pJ1dgrBa; ctid=20; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; fzq_js_anjuke_xiaoqu_m=c80eeca4a3301dbf27b3c79b9f18fa3b_1652536277223_27; xzuid=81992155-5990-4a2f-8842-32bcbfbb1117")
|
||||||
|
header.put("referer","https://m.anjuke.com/fz/")
|
||||||
|
header.put("sec-fetch-dest","empty")
|
||||||
|
header.put("sec-fetch-mode","cors")
|
||||||
|
header.put("sec-fetch-site","same-origin")
|
||||||
|
header.put("sw8","1-ODA0ZDA3N2ZmMDA1NDgyYmExNTFlODhlZWIyNmFkMzk=-N2UwZDkxMGU2ZTdjNDdkZmJlZDBkMjUzMmNkODQ2NWU=-0-MTE3-aHR0cHM6Ly9tLmFuanVrZS5jb20=-L2VzZi1hamF4L2NvbW11bml0eS9saXN0-bS5hbmp1a2UuY29t-1")
|
||||||
|
header.put("sw8-correlation","c291cmNl:YmVpZG91")
|
||||||
|
header.put("user-agent","Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1")
|
||||||
|
|
||||||
|
def doc = ABuYunUtil.doGet(url, null, header, "utf-8")
|
||||||
|
def links = []
|
||||||
|
if(doc){
|
||||||
|
def eles = JQueryUtils.select(".column2 a", doc)
|
||||||
|
for(def e : eles){
|
||||||
|
def href = e.attr("href")
|
||||||
|
links << href
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return links;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def run(){
|
||||||
|
//"仓山":4071,
|
||||||
|
def map = ["晋安":4070,"鼓楼":4068,"闽侯":4073,"台江":4069,"福清":16378,"长乐":15880,"马尾":4072,"连江":16380,"平潭":16387,"罗源":16382,"永泰":16392,"闽清":16384]
|
||||||
|
map.each{region, area_id->
|
||||||
|
for(def i=1;i<40;i++){
|
||||||
|
def stop = false;
|
||||||
|
def all_stop = false
|
||||||
|
while(!stop){//渝北:2359 南岸:2368 沙坪坝:2380
|
||||||
|
//城口
|
||||||
|
def flagMap = getInfo(stop, i, area_id)
|
||||||
|
stop = flagMap.stop
|
||||||
|
all_stop = flagMap.all_stop
|
||||||
|
}
|
||||||
|
if(all_stop){
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def getInfo(def stop, def i, def area_id){
|
||||||
|
|
||||||
|
def map = [:]
|
||||||
|
def url = "https://m.anjuke.com/esf-ajax/community/list?city_id=35&page_size=100&page=${i}&area_id=${area_id}&is_tw=1"
|
||||||
|
|
||||||
|
def header = [:]
|
||||||
|
header.put("accept","application/json, text/plain, */*")
|
||||||
|
header.put("accept-encoding","gzip, deflate, br")
|
||||||
|
header.put("accept-language","zh-CN,zh;q=0.9")
|
||||||
|
header.put("cookie","aQQ_ajkguid=E9796514-38C0-44D2-8CA3-1C58C3928B29; sessid=765EEB9D-E7A2-4B2C-8151-CA9CD939AB02; id58=CpQB22J8sqMA9stuJfeAAg==; twe=2; 58tj_uuid=bf6eaff8-851c-471a-a914-853242878ddc; als=0; seo_source_type=1; _ga=GA1.2.215255179.1652344695; isp=true; ajk-appVersion=; obtain_by=2; fzq_js_anjuke_ershoufang_m=c2c12b2d1b221734fa20ef752cd6bdc0_1652509223606_27; fzq_h=ef0186eeaba082ecd2a47ede7c7e3e64_1652533612190_a34c026e9c1842a3b35bdd37f3d2aa0b_2015773878; init_refer=https%253A%252F%252Fm.anjuke.com%252Fcq%252Fcommunity%252F; new_uv=8; wmda_uuid=b59ce2efbba070ecfa8fe93d79f30dd4; wmda_new_uuid=1; wmda_session_id_6145577459763=1652535525740-dae51249-3379-d4e7; wmda_visited_projects=%3B6145577459763; new_session=0; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; xxzl_deviceid=q9XQbdA9hhkHJTj+RgVpwc7jrBzVJp9J1uTlp4hDtRiXtA615K4HS4R0pJ1dgrBa; ctid=20; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; fzq_js_anjuke_xiaoqu_m=c80eeca4a3301dbf27b3c79b9f18fa3b_1652536277223_27; xzuid=81992155-5990-4a2f-8842-32bcbfbb1117")
|
||||||
|
header.put("referer","https://m.anjuke.com/fz/community/")
|
||||||
|
header.put("sec-fetch-dest","empty")
|
||||||
|
header.put("sec-fetch-mode","cors")
|
||||||
|
header.put("sec-fetch-site","same-origin")
|
||||||
|
header.put("sw8","1-ODA0ZDA3N2ZmMDA1NDgyYmExNTFlODhlZWIyNmFkMzk=-N2UwZDkxMGU2ZTdjNDdkZmJlZDBkMjUzMmNkODQ2NWU=-0-MTE3-aHR0cHM6Ly9tLmFuanVrZS5jb20=-L2VzZi1hamF4L2NvbW11bml0eS9saXN0-bS5hbmp1a2UuY29t-1")
|
||||||
|
header.put("sw8-correlation","c291cmNl:YmVpZG91")
|
||||||
|
header.put("user-agent","Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1")
|
||||||
|
|
||||||
|
def ret = ABuYunUtil.doGet(url, null, null, "utf-8")
|
||||||
|
def all_stop = false;
|
||||||
|
if (ret) {
|
||||||
|
JSONObject jsonObj = JSONObject.fromObject(ret);
|
||||||
|
if(jsonObj && jsonObj.status=="ok" ){
|
||||||
|
def data = jsonObj.data
|
||||||
|
if(data && data.total.toInteger()>0){
|
||||||
|
def communities = data.communities
|
||||||
|
|
||||||
|
println "【${i}】 "+communities.size() +" "+url
|
||||||
|
if(communities.size()>0){
|
||||||
|
save("福州", communities)
|
||||||
|
Thread.sleep(10000)
|
||||||
|
stop = true;
|
||||||
|
}else{
|
||||||
|
stop = true;
|
||||||
|
println url +" 返回0"
|
||||||
|
all_stop = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
}else{
|
||||||
|
stop = false;
|
||||||
|
println url +" 无数据"
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}else{
|
||||||
|
stop = false;
|
||||||
|
Thread.sleep(5000)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
map.stop = stop
|
||||||
|
map.all_stop = all_stop;
|
||||||
|
return map;
|
||||||
|
}
|
||||||
|
|
||||||
|
def save(city, def ret){
|
||||||
|
def index = 0
|
||||||
|
for(def o : ret){
|
||||||
|
index++
|
||||||
|
def obj = findIsExist(o.jumpAction)
|
||||||
|
if(!obj){
|
||||||
|
def udate = new BasicDBObject()
|
||||||
|
o.each{k,v->
|
||||||
|
udate.put(k,v)
|
||||||
|
}
|
||||||
|
udate.put("city", city)
|
||||||
|
udate.put("c_date", new Date())
|
||||||
|
detail.save(udate)
|
||||||
|
println index+" [保存]"+o.jumpAction
|
||||||
|
}else{
|
||||||
|
println index+" [已存在]"+o.jumpAction
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def findIsExist(def url){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("jumpAction",url)
|
||||||
|
return detail.findOne(q);
|
||||||
|
}
|
||||||
|
}
|
||||||
+173
-61
@@ -11,9 +11,15 @@ import org.openqa.selenium.By
|
|||||||
import org.openqa.selenium.WebDriver
|
import org.openqa.selenium.WebDriver
|
||||||
import org.openqa.selenium.WebElement
|
import org.openqa.selenium.WebElement
|
||||||
import org.openqa.selenium.chrome.ChromeDriver
|
import org.openqa.selenium.chrome.ChromeDriver
|
||||||
|
import org.openqa.selenium.remote.DesiredCapabilities
|
||||||
|
import org.openqa.selenium.remote.RemoteWebDriver
|
||||||
|
|
||||||
class AnjukeDetailCrawl {
|
class AnjukeDetailCrawl {
|
||||||
|
|
||||||
DBCollection detailColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_detail")
|
DBCollection detailColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_detail")
|
||||||
|
|
||||||
|
DBCollection urlColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "detail_url")
|
||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new AnjukeDetailCrawl();
|
def task = new AnjukeDetailCrawl();
|
||||||
task.run();
|
task.run();
|
||||||
@@ -21,9 +27,15 @@ class AnjukeDetailCrawl {
|
|||||||
def run(){
|
def run(){
|
||||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||||
System.setProperty("http.nonProxyHosts", "localhost");
|
System.setProperty("http.nonProxyHosts", "localhost");
|
||||||
def url = "https://chongqing.anjuke.com/community/?from=navigation"
|
def url = "https://chongqing.anjuke.com/"
|
||||||
WebDriver driver = new ChromeDriver()
|
WebDriver driver = new ChromeDriver()
|
||||||
|
// DesiredCapabilities capabilities = DesiredCapabilities.chrome();
|
||||||
|
// URL urlInstance = new URL("http://192.168.0.100:30025"); //指定URL
|
||||||
|
// WebDriver driver = new RemoteWebDriver(urlInstance, capabilities);
|
||||||
|
|
||||||
|
driver.manage().deleteAllCookies()
|
||||||
def regions = getRegions(driver, url)
|
def regions = getRegions(driver, url)
|
||||||
|
println "区域链接: "+regions.size()
|
||||||
getAllRegions(driver, regions)
|
getAllRegions(driver, regions)
|
||||||
|
|
||||||
}
|
}
|
||||||
@@ -31,44 +43,88 @@ class AnjukeDetailCrawl {
|
|||||||
|
|
||||||
def getAllRegions(driver, regions){
|
def getAllRegions(driver, regions){
|
||||||
for(def url : regions){
|
for(def url : regions){
|
||||||
driver.get(url)
|
def doc = getDocument(url, driver)
|
||||||
def doc = Jsoup.parse(driver.getPageSource());
|
|
||||||
def total = getTotal(doc)
|
def total = getTotal(doc)
|
||||||
if(total) {
|
if(total) {
|
||||||
if (total > 1250) {
|
if (total > 1250) {
|
||||||
|
def links = getMoreLinks(url, driver)
|
||||||
|
println url +" "+links.size()
|
||||||
|
saveDetailLinks(links)
|
||||||
|
// parse(links,driver)
|
||||||
} else {
|
} else {
|
||||||
def links = getAllLinks(url, driver)
|
def links = getAllLinks(url, driver)
|
||||||
parse(links,driver)
|
println url +" "+links.size()
|
||||||
|
saveDetailLinks(links)
|
||||||
|
// parse(links,driver)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def saveDetailLinks(links){
|
||||||
|
for(def url : links){
|
||||||
|
def data = new BasicDBObject()
|
||||||
|
data.put("city","重庆")
|
||||||
|
data.put("url", url)
|
||||||
|
data.put("source", "安居客")
|
||||||
|
data.put("c_date", new Date())
|
||||||
|
urlColl.save(data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def getMoreLinks(url, driver){
|
||||||
|
def doc = Jsoup.parse(driver.getPageSource());
|
||||||
|
def pricelinks = JQueryUtils.select(".filter-wrap:contains(均价) .line-item a", doc)
|
||||||
|
def links = []
|
||||||
|
if(pricelinks){
|
||||||
|
for(def link : pricelinks){
|
||||||
|
if(link.text()!="不限"){
|
||||||
|
def urlLink = link.attr("href")
|
||||||
|
getDocument(urlLink, driver)
|
||||||
|
// driver.get(urlLink)
|
||||||
|
links.addAll(getAllLinks(urlLink, driver))
|
||||||
|
println urlLink +" "+links.size()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return links;
|
||||||
|
}
|
||||||
|
|
||||||
def getAllLinks(url, driver){
|
def getAllLinks(url, driver){
|
||||||
def stop = false
|
def stop = false
|
||||||
def urls = []
|
def urls = []
|
||||||
while(!stop) {
|
def doc = Jsoup.parse(driver.getPageSource());
|
||||||
def doc = Jsoup.parse(driver.getPageSource());
|
def total = getTotal(doc)
|
||||||
|
def page = getPage(total, 25)
|
||||||
|
def pages = []
|
||||||
|
for(def i=1;i<=page;i++){
|
||||||
|
pages << url.substring(0, url.size()-1)+"-p"+i+"/"
|
||||||
|
}
|
||||||
|
for(def pageUrl : pages){
|
||||||
|
doc = getDocument(pageUrl, driver)
|
||||||
def details = JQueryUtils.select(".list-cell a.li-row", doc)
|
def details = JQueryUtils.select(".list-cell a.li-row", doc)
|
||||||
|
|
||||||
for (def d : details) {
|
for (def d : details) {
|
||||||
urls << d.attr("href")
|
def link = d.attr("href")
|
||||||
}
|
if(!urls.contains(link)){
|
||||||
def nextEle = JQueryUtils.select(".next", doc)
|
urls << d.attr("href")
|
||||||
if (nextEle) {
|
}
|
||||||
driver.findElement(By.cssSelector(".next")).click()
|
|
||||||
}else{
|
|
||||||
stop = true
|
|
||||||
}
|
|
||||||
if(urls.size()>20){
|
|
||||||
stop = true
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return urls
|
return urls
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def getPage(def total, def pageSize){
|
||||||
|
if(total && total.toInteger()>0){
|
||||||
|
def page = total.toInteger()/pageSize;
|
||||||
|
if(page>(int)page){
|
||||||
|
page = (int)page + 1
|
||||||
|
return page
|
||||||
|
}
|
||||||
|
return (int)page;
|
||||||
|
}
|
||||||
|
return null
|
||||||
|
}
|
||||||
def getTotal(doc){
|
def getTotal(doc){
|
||||||
|
|
||||||
def totalText = JQueryUtils.selectText(".total-info", doc)
|
def totalText = JQueryUtils.selectText(".total-info", doc)
|
||||||
@@ -82,9 +138,19 @@ class AnjukeDetailCrawl {
|
|||||||
}
|
}
|
||||||
|
|
||||||
def getRegions(driver, url){
|
def getRegions(driver, url){
|
||||||
|
// def doc = getDocument(url, driver)
|
||||||
driver.get(url)
|
driver.get(url)
|
||||||
Thread.sleep(3000)
|
|
||||||
def doc = Jsoup.parse(driver.getPageSource());
|
def doc = Jsoup.parse(driver.getPageSource());
|
||||||
|
def els = JQueryUtils.select("a:contains(小区)", doc)
|
||||||
|
if(els){
|
||||||
|
def href = els[0].attr("href")
|
||||||
|
doc = Jsoup.parse(driver.getPageSource());
|
||||||
|
// driver.findElement(By.xpath("//a[contains(text(), '小区')]")).click()
|
||||||
|
// driver.findElement(By.linkText("小区")).click()
|
||||||
|
doc = getDocument(href, driver)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
// def loginIfm = JQueryUtils.select("#iframeLoginIfm", doc)
|
// def loginIfm = JQueryUtils.select("#iframeLoginIfm", doc)
|
||||||
// if(loginIfm){
|
// if(loginIfm){
|
||||||
// def href = loginIfm[0].attr("src")
|
// def href = loginIfm[0].attr("src")
|
||||||
@@ -107,23 +173,35 @@ class AnjukeDetailCrawl {
|
|||||||
def regionEles = JQueryUtils.select(".region-parents li a", doc)
|
def regionEles = JQueryUtils.select(".region-parents li a", doc)
|
||||||
def urls = []
|
def urls = []
|
||||||
for(def e : regionEles){
|
for(def e : regionEles){
|
||||||
urls << e.attr("href")
|
if(e.text()!="不限"){
|
||||||
|
urls << e.attr("href")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return urls;
|
return urls;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def getNontCheckPage(url, driver, doc){
|
||||||
|
def stop = false;
|
||||||
|
while(!stop){
|
||||||
|
if(doc){
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return doc;
|
||||||
|
}
|
||||||
|
|
||||||
def parse(def links, driver){
|
def parse(def links, driver){
|
||||||
def rets = []
|
def rets = []
|
||||||
for(def link : links){
|
for(def link : links){
|
||||||
def district = parsing(link, driver)
|
def district = parsing(link, driver)
|
||||||
if(district){
|
if(district){
|
||||||
rets << district
|
// rets << district
|
||||||
|
|
||||||
if (rets.size() % 100 == 0) {
|
// if (rets.size() % 100 == 0) {
|
||||||
def add= saveData(rets)
|
saveObj(district)
|
||||||
println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
println district.name+" "+district.url + " "+DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||||
rets = []
|
// rets = []
|
||||||
}
|
// }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (rets.size()> 0) {
|
if (rets.size()> 0) {
|
||||||
@@ -136,33 +214,61 @@ class AnjukeDetailCrawl {
|
|||||||
def saveData(def results){
|
def saveData(def results){
|
||||||
def add = 0
|
def add = 0
|
||||||
for(def ret : results){
|
for(def ret : results){
|
||||||
def q = new BasicDBObject()
|
saveObj(ret)
|
||||||
q.put("url", ret.url)
|
|
||||||
def dbo = detailColl.findOne(q);
|
|
||||||
|
|
||||||
DBObject obj = new BasicDBObject();
|
|
||||||
ret.each{k,v->
|
|
||||||
obj.put(k,v)
|
|
||||||
}
|
|
||||||
|
|
||||||
if(dbo){
|
|
||||||
obj.m_date = new Date()
|
|
||||||
detailColl.update(q, new BasicDBObject('$set', obj))
|
|
||||||
}else{
|
|
||||||
add++
|
|
||||||
obj.c_date = new Date()
|
|
||||||
obj.m_date = new Date()
|
|
||||||
try{
|
|
||||||
detailColl.save(obj)
|
|
||||||
}catch(Exception e){
|
|
||||||
println obj.url
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return add
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def saveObj(def ret){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("url", ret.url)
|
||||||
|
def dbo = detailColl.findOne(q);
|
||||||
|
|
||||||
|
DBObject obj = new BasicDBObject();
|
||||||
|
ret.each{k,v->
|
||||||
|
obj.put(k,v)
|
||||||
|
}
|
||||||
|
|
||||||
|
if(dbo){
|
||||||
|
obj.m_date = new Date()
|
||||||
|
detailColl.update(q, new BasicDBObject('$set', obj))
|
||||||
|
}else{
|
||||||
|
|
||||||
|
obj.c_date = new Date()
|
||||||
|
obj.m_date = new Date()
|
||||||
|
try{
|
||||||
|
detailColl.save(obj)
|
||||||
|
}catch(Exception e){
|
||||||
|
println obj.url
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
def getDocument(url, driver){
|
||||||
|
def stop = false
|
||||||
|
def error = 0
|
||||||
|
def doc = null
|
||||||
|
while(!stop){
|
||||||
|
try{
|
||||||
|
driver.get(url)
|
||||||
|
doc = Jsoup.parse(driver.getPageSource());
|
||||||
|
if(doc && !doc.toString().contains("访问验证-安居客") && !doc.toString().contains("502 Bad Gateway") && !doc.toString().contains("<title>登录</title>")){
|
||||||
|
stop = true;
|
||||||
|
}else{
|
||||||
|
error++
|
||||||
|
driver.manage().deleteAllCookies()
|
||||||
|
println "[${error}] error: "+url
|
||||||
|
Thread.sleep(5000)
|
||||||
|
}
|
||||||
|
}catch(Exception e){
|
||||||
|
error++
|
||||||
|
}
|
||||||
|
if(error>=20){
|
||||||
|
stop = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return doc
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def parsing(url, driver){
|
def parsing(url, driver){
|
||||||
|
|
||||||
@@ -191,10 +297,9 @@ class AnjukeDetailCrawl {
|
|||||||
|
|
||||||
def infoDiv = JQueryUtils.select(".info-list", doc)
|
def infoDiv = JQueryUtils.select(".info-list", doc)
|
||||||
district.estate_type = getValue(infoDiv.select(".column-2:contains(物业类型) .value").text())
|
district.estate_type = getValue(infoDiv.select(".column-2:contains(物业类型) .value").text())
|
||||||
district.estate_charge = getValue(infoDiv.select(".column-2:contains(物业费) .value").text())
|
district.quanshu_type = getValue(infoDiv.select(".column-2:contains(权属类别) .value").text())
|
||||||
district.estate_company = getValue(infoDiv.select(".column-1:contains(物业公司) .value").text())
|
district.finish_date = getValue(infoDiv.select(".column-2:contains(竣工时间) .value").text())
|
||||||
district.developer = getValue(infoDiv.select(".column-1:contains(开发商) .value").text())
|
district.property_life = getValue(infoDiv.select(".column-2:contains(产权年限) .value").text())
|
||||||
|
|
||||||
def houseCountStr = infoDiv.select(".column-2:contains(总户数) .value").text()
|
def houseCountStr = infoDiv.select(".column-2:contains(总户数) .value").text()
|
||||||
if (houseCountStr) {
|
if (houseCountStr) {
|
||||||
houseCountStr = houseCountStr.replace("户", "")
|
houseCountStr = houseCountStr.replace("户", "")
|
||||||
@@ -202,9 +307,13 @@ class AnjukeDetailCrawl {
|
|||||||
district.house_count = houseCountStr.toInteger()
|
district.house_count = houseCountStr.toInteger()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
district.finish_date = getValue(infoDiv.select(".column-2:contains(竣工时间) .value").text())
|
|
||||||
district.build_area = getValue(infoDiv.select(".column-2:contains(总建面积) .value").text())
|
district.build_area = getValue(infoDiv.select(".column-2:contains(总建面积) .value").text())
|
||||||
|
district.volumetric_rate = getValue(infoDiv.select(".column-2:contains(容积率) .value").text())
|
||||||
|
district.greening_rate = getValue(infoDiv.select(".column-2:contains(绿化率) .value").text())
|
||||||
|
|
||||||
|
district.build_type = getValue(infoDiv.select(".column-2:contains(建筑类型) .value").text())
|
||||||
|
district.business = getValue(infoDiv.select(".column-2:contains(所属商圈) .value").text())
|
||||||
|
|
||||||
def carSiteStr = getValue(infoDiv.select(".column-2:contains(停车位) .value").text())
|
def carSiteStr = getValue(infoDiv.select(".column-2:contains(停车位) .value").text())
|
||||||
if(carSiteStr){
|
if(carSiteStr){
|
||||||
if(carSiteStr.contains("(")){
|
if(carSiteStr.contains("(")){
|
||||||
@@ -214,16 +323,19 @@ class AnjukeDetailCrawl {
|
|||||||
district.car_site = carSiteStr
|
district.car_site = carSiteStr
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
district.volumetric_rate = getValue(infoDiv.select(".column-2:contains(容积率) .value").text())
|
district.car_site_str = infoDiv.select(".column-2:contains(停车位) .value").text()
|
||||||
district.greening_rate = getValue(infoDiv.select(".column-2:contains(绿化率:) .value").text())
|
district.estate_charge = getValue(infoDiv.select(".column-2:contains(物业费) .value").text())
|
||||||
district.build_type = getValue(infoDiv.select(".column-2:contains(建筑类型) .value").text())
|
|
||||||
district.property_life = getValue(infoDiv.select(".column-2:contains(产权年限) .value").text())
|
district.stop_charge = getValue(infoDiv.select(".column-1:contains(停车费) .value").text())
|
||||||
district.car_site_charge = getValue(infoDiv.select(".column-1:contains(停车费) .value").text())
|
district.car_site_charge = getValue(infoDiv.select(".column-1:contains(车位管理费) .value").text())
|
||||||
|
|
||||||
|
district.estate_company = getValue(infoDiv.select(".column-1:contains(物业公司) .value").text())
|
||||||
|
district.developer = getValue(infoDiv.select(".column-1:contains(开发商) .value").text())
|
||||||
district.url = url
|
district.url = url
|
||||||
district.c_date = new Date()
|
district.c_date = new Date()
|
||||||
district.m_date = new Date()
|
district.m_date = new Date()
|
||||||
|
district.crawl_type = "chrome"
|
||||||
district.root = "anjuke.com"
|
district.root = "anjuke.com"
|
||||||
|
|
||||||
stop = true;
|
stop = true;
|
||||||
}else{
|
}else{
|
||||||
Thread.sleep(3000)
|
Thread.sleep(3000)
|
||||||
|
|||||||
+89
@@ -0,0 +1,89 @@
|
|||||||
|
package com.fangjia.script.gj.crawl
|
||||||
|
|
||||||
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
|
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
|
||||||
|
import com.fangjia.script.util.ExcelUtils
|
||||||
|
import com.fangjia.script.util.OpenApiClient
|
||||||
|
|
||||||
|
class AnjukeDistrictMatch {
|
||||||
|
|
||||||
|
static main(args) {
|
||||||
|
def task = new AnjukeDistrictMatch();
|
||||||
|
task.run();
|
||||||
|
// task.run1()
|
||||||
|
}
|
||||||
|
|
||||||
|
def run1(){
|
||||||
|
def regionPointsMap = findRegionRange("重庆")
|
||||||
|
def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region))
|
||||||
|
}
|
||||||
|
def run(){
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def filePath = "D:\\数据\\重庆农商行\\安居客\\重庆小区.xlsx"
|
||||||
|
ExcelUtils.multiThreadingModify(filePath,{row->
|
||||||
|
if(row['A'] && !row['J']){
|
||||||
|
def city = row['B']
|
||||||
|
def region = row['C']
|
||||||
|
def name = row['D']
|
||||||
|
def address = row['E']
|
||||||
|
// def flag = "别名匹配"
|
||||||
|
|
||||||
|
|
||||||
|
// def dist = matchDist(city, null, name)
|
||||||
|
// if(!row['J'] && name.contains("(")){
|
||||||
|
// name = name.replace("(${region})","").replace("(","").replace(")","")
|
||||||
|
// flag = "去括号匹配"
|
||||||
|
// dist = matchDist(city, region, name)
|
||||||
|
// }
|
||||||
|
// if(!dist){
|
||||||
|
// dist = CommonDatautils.findSeaweedByMappingAlias(city, region, name,"district")
|
||||||
|
// flag = "映射别名匹配"
|
||||||
|
// }
|
||||||
|
// if(!dist) {
|
||||||
|
// def rets = CommonDatautils.findDistrictByAlias(city, region, name, "district")
|
||||||
|
// if(rets && rets.size()==1){
|
||||||
|
// dist = rets[0]
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
//
|
||||||
|
// if(!dist){
|
||||||
|
// def dists = CommonDatautils.findDistrictByAddress(city, region, address,"district")
|
||||||
|
// flag = "地址匹配"
|
||||||
|
// if(dists.size()>1){
|
||||||
|
// println address +" 地址相同:"+dists.size()
|
||||||
|
// }else{
|
||||||
|
// if(dists.size()==1){
|
||||||
|
// dist = dists[0]
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
// if(dist){
|
||||||
|
// row['J'] = flag
|
||||||
|
// row['K'] = dist._id.toString()
|
||||||
|
// row['L'] = dist.region
|
||||||
|
// row['M'] = dist.name
|
||||||
|
// row['N'] = dist.address
|
||||||
|
// }
|
||||||
|
|
||||||
|
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false,region,name)
|
||||||
|
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||||
|
// row['J'] = "匹配接口"
|
||||||
|
// row['K'] = jsonObject.get("id")
|
||||||
|
// row['L'] = jsonObject.get("district")
|
||||||
|
// row['M'] = jsonObject.get("name")
|
||||||
|
// row['N'] = jsonObject.get("address")
|
||||||
|
row['M'] = jsonObject.get("name")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
def matchDist(city, region, name){
|
||||||
|
def dist = CommonDatautils.findDistrict(city, region, name, "district")
|
||||||
|
return dist;
|
||||||
|
}
|
||||||
|
}
|
||||||
+628
@@ -0,0 +1,628 @@
|
|||||||
|
package com.fangjia.script.gj.crawl
|
||||||
|
|
||||||
|
import com.fangjia.core.util.BaiDuApiUtils
|
||||||
|
import com.fangjia.core.util.GeoUtils
|
||||||
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
|
import com.fangjia.script.linxsh.util.AddressUtil
|
||||||
|
import com.fangjia.script.util.CommonUtils
|
||||||
|
import com.fangjia.script.util.DateUtils
|
||||||
|
import com.fangjia.script.util.DbUtils
|
||||||
|
import com.fangjia.script.util.ExcelUtils
|
||||||
|
import com.fangjia.script.util.MongoUtils
|
||||||
|
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||||
|
import com.mongodb.BasicDBObject
|
||||||
|
import com.mongodb.DBCollection
|
||||||
|
import com.rabbitmq.client.impl.AMQImpl
|
||||||
|
import net.sf.json.JSONObject
|
||||||
|
import org.bson.types.ObjectId
|
||||||
|
|
||||||
|
class AnjukeToDB {
|
||||||
|
|
||||||
|
DBCollection detail = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "app_anjuke_detail")
|
||||||
|
|
||||||
|
static main(args) {
|
||||||
|
def task = new AnjukeToDB();
|
||||||
|
// task.run();
|
||||||
|
// task.addNoMatch()
|
||||||
|
task.updateEstateType()
|
||||||
|
// task.gaodeRangeData()
|
||||||
|
}
|
||||||
|
|
||||||
|
def updateEstateType(){
|
||||||
|
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\安居客重庆小区-曹玉彦.xlsx",0,1)
|
||||||
|
for(def row : datas){
|
||||||
|
def anjukeId = row['A']
|
||||||
|
def obj = findAnjukeDistrictDetail(anjukeId)
|
||||||
|
if(obj) {
|
||||||
|
def type = getEstatType(obj.extend.type)
|
||||||
|
def commId = row['L']
|
||||||
|
if(!commId){
|
||||||
|
commId = row['G']
|
||||||
|
}
|
||||||
|
if(type){
|
||||||
|
if(type=="别墅"){
|
||||||
|
def udata = new BasicDBObject()
|
||||||
|
udata.put("estate_type2", type)
|
||||||
|
udata.put("m_date", new Date())
|
||||||
|
println commId
|
||||||
|
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(commId.trim())), new BasicDBObject('$set', udata))
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
// def udata = new BasicDBObject()
|
||||||
|
// udata.put("status", 1)
|
||||||
|
// udata.put("name_cat",11)
|
||||||
|
// udata.put("m_date", new Date())
|
||||||
|
// DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(commId)), new BasicDBObject('$set', udata))
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def gaodeRangeData(){
|
||||||
|
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\徐鹤高德地图.xlsx",0,1)
|
||||||
|
def index = 0
|
||||||
|
for(def d : datas){
|
||||||
|
index++
|
||||||
|
println index
|
||||||
|
def personName = d['AB']
|
||||||
|
if(personName){
|
||||||
|
def commid = d['J']
|
||||||
|
if(commid){
|
||||||
|
def obj = CommonDatautils.findDistrict("重庆",d['C'],personName,"district")
|
||||||
|
if(obj){
|
||||||
|
updateDist(d, obj._id.toString())
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
saveGaode(d)
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
def commid = d['J']
|
||||||
|
updateDist(d, commid)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def saveGaode(d){
|
||||||
|
def oldName = d['E']
|
||||||
|
def distName = nameFileter(d['C'], oldName)
|
||||||
|
def udata = new BasicDBObject()
|
||||||
|
def alias = []
|
||||||
|
|
||||||
|
def aliasStr = d['F']
|
||||||
|
if(aliasStr){
|
||||||
|
def arrs = aliasStr.split("/")
|
||||||
|
for(def s : arrs){
|
||||||
|
alias << s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(alias.size()>0){
|
||||||
|
udata.put("alias", alias)
|
||||||
|
}
|
||||||
|
|
||||||
|
def address = AddressUtil.formatAddress("重庆", d['G'])
|
||||||
|
if(address){
|
||||||
|
udata.put("address", address)
|
||||||
|
}
|
||||||
|
if(d['N']){
|
||||||
|
udata.put("virescence_percent", d['N'])
|
||||||
|
}
|
||||||
|
|
||||||
|
if(d['O']){
|
||||||
|
udata.put("estate_charge", d['O'])
|
||||||
|
}
|
||||||
|
if(d['Q']){
|
||||||
|
udata.put("cubage_percent", d['Q'])
|
||||||
|
}
|
||||||
|
if(d['R']){
|
||||||
|
udata.put("developer", d['R'])
|
||||||
|
}
|
||||||
|
|
||||||
|
if( d['S']){
|
||||||
|
udata.put("build_area", d['S'])
|
||||||
|
}
|
||||||
|
|
||||||
|
if(d['T']){
|
||||||
|
def t = d['T'].substring(0, 3)
|
||||||
|
udata.put("build_date", t)
|
||||||
|
}
|
||||||
|
|
||||||
|
if(d['V'] && d['V'].isNumber()){
|
||||||
|
udata.put("car_site_count", d['V'].toInteger())
|
||||||
|
}
|
||||||
|
|
||||||
|
if(d['Y']){
|
||||||
|
def build_type = getBuildType2(d['Y'])
|
||||||
|
if(build_type){
|
||||||
|
udata.put("build_type", build_type)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if(d['Z']){
|
||||||
|
udata.put("property_life", d['Z'])
|
||||||
|
}
|
||||||
|
|
||||||
|
def lat = null
|
||||||
|
def lng = null
|
||||||
|
if(d['H']){
|
||||||
|
lat = d['H']
|
||||||
|
lng = d['I']
|
||||||
|
// 高德转百度
|
||||||
|
def baidu = BaiDuApiUtils.geoconv(lng.toDouble(), lat.toDouble(), 3)
|
||||||
|
def lng2 = baidu.x
|
||||||
|
def lat2 = baidu.y
|
||||||
|
|
||||||
|
Long dLat2 = lat2 * 10000000000L
|
||||||
|
Long dLng2 = lng2 * 10000000000L
|
||||||
|
udata.put("lat2", dLat2)
|
||||||
|
udata.put("lng2", dLng2)
|
||||||
|
Long dlat = lat.toDouble() * 10000000000L
|
||||||
|
Long dlng = lng.toDouble() * 10000000000L
|
||||||
|
udata.put("lat", dlat)
|
||||||
|
udata.put("lng", dlng)
|
||||||
|
}
|
||||||
|
def type = "普通住宅"
|
||||||
|
if(distName.contains("别墅")) {
|
||||||
|
type = "别墅"
|
||||||
|
}else if(distName.contains("商住")){
|
||||||
|
type = "商住"
|
||||||
|
}
|
||||||
|
|
||||||
|
if(udata){
|
||||||
|
udata.put("name_cat",0)
|
||||||
|
udata.put("status",0)
|
||||||
|
udata.put("cat","district")
|
||||||
|
udata.put("c_date", new Date())
|
||||||
|
udata.put("m_date", new Date())
|
||||||
|
udata.put("city", "重庆")
|
||||||
|
udata.put("region",d['C'])
|
||||||
|
udata.put("name", distName)
|
||||||
|
udata.put("estate_type2",type)
|
||||||
|
|
||||||
|
def markMap = [:]
|
||||||
|
markMap.put("data_source", "机器入库")
|
||||||
|
udata.put("mark_map",markMap)
|
||||||
|
DbUtils.seaweed.save(udata);
|
||||||
|
|
||||||
|
def stop = false
|
||||||
|
while (!stop){
|
||||||
|
if(udata._id.toString()){
|
||||||
|
addPrice(udata)
|
||||||
|
stop = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def updateDist(d, commid){
|
||||||
|
|
||||||
|
if(commid){
|
||||||
|
def dist = CommonDatautils.queryDistrictById(commid)
|
||||||
|
if(dist){
|
||||||
|
def oldName = d['E']
|
||||||
|
def distName = nameFileter(d['C'], oldName)
|
||||||
|
def udata = new BasicDBObject()
|
||||||
|
if(!distName.contains("号")){
|
||||||
|
def alias = dist.alias
|
||||||
|
if(!alias){
|
||||||
|
alias = []
|
||||||
|
}
|
||||||
|
if(!alias.contains(distName) && dist.name!=distName){
|
||||||
|
alias << distName
|
||||||
|
}
|
||||||
|
def aliasStr = d['F']
|
||||||
|
if(aliasStr){
|
||||||
|
def arrs = aliasStr.split("/")
|
||||||
|
for(def s : arrs){
|
||||||
|
alias << s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(alias.size()>0 && alias!=dist.alias){
|
||||||
|
udata.put("alias", alias)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if(dist.address && !dist.address.contains("号")){
|
||||||
|
def address = AddressUtil.formatAddress("重庆", d['G'])
|
||||||
|
if(address){
|
||||||
|
udata.put("address", address)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if(!dist.virescence_percent && d['N']){
|
||||||
|
udata.put("virescence_percent", d['N'])
|
||||||
|
}
|
||||||
|
|
||||||
|
if(!dist.estate_charge && d['O']){
|
||||||
|
udata.put("estate_charge", d['O'])
|
||||||
|
}
|
||||||
|
if(!dist.cubage_percent && d['Q']){
|
||||||
|
udata.put("cubage_percent", d['Q'])
|
||||||
|
}
|
||||||
|
if(!dist.developer && d['R']){
|
||||||
|
udata.put("developer", d['R'])
|
||||||
|
}
|
||||||
|
|
||||||
|
if(!dist.build_area && d['S']){
|
||||||
|
udata.put("build_area", d['S'])
|
||||||
|
}
|
||||||
|
|
||||||
|
if(!dist.build_date && d['T']){
|
||||||
|
if(d['T'].size()>=4){
|
||||||
|
def t = d['T'].substring(0, 4)
|
||||||
|
udata.put("build_date", t)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
if(!dist.car_site_count && d['V'] && d['V'].isNumber()){
|
||||||
|
udata.put("car_site_count", d['V'].toInteger())
|
||||||
|
}
|
||||||
|
|
||||||
|
if(!dist.build_type && d['Y']){
|
||||||
|
def build_type = getBuildType2(d['Y'])
|
||||||
|
if(build_type){
|
||||||
|
udata.put("build_type", build_type)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if(!dist.property_life && d['Z']){
|
||||||
|
udata.put("property_life", d['Z'])
|
||||||
|
}
|
||||||
|
|
||||||
|
def lat = null
|
||||||
|
def lng = null
|
||||||
|
if(d['H']){
|
||||||
|
lat = d['H']
|
||||||
|
lng = d['I']
|
||||||
|
|
||||||
|
def slat = dist.lat/10000000000.0
|
||||||
|
def slng = dist.lng/10000000000.0
|
||||||
|
|
||||||
|
def distance = GeoUtils.calculateDistance(lng.toDouble(), lat.toDouble(), slng, slat);
|
||||||
|
if(distance>1000){
|
||||||
|
// 高德转百度
|
||||||
|
def baidu = BaiDuApiUtils.geoconv(lng.toDouble(), lat.toDouble(), 3)
|
||||||
|
def lng2 = baidu.x
|
||||||
|
def lat2 = baidu.y
|
||||||
|
|
||||||
|
Long dLat2 = lat2 * 10000000000L
|
||||||
|
Long dLng2 = lng2 * 10000000000L
|
||||||
|
udata.put("lat2", dLat2)
|
||||||
|
udata.put("lng2", dLng2)
|
||||||
|
Long dlat = lat.toDouble() * 10000000000L
|
||||||
|
Long dlng = lng.toDouble() * 10000000000L
|
||||||
|
udata.put("lat", dlat)
|
||||||
|
udata.put("lng", dlng)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
if(udata){
|
||||||
|
udata.put("m_date", new Date())
|
||||||
|
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||||
|
println "[修改] ${dist.name}"+ udata
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def addNoMatch(){
|
||||||
|
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\安居客未匹配.xlsx",0,1)
|
||||||
|
for(def d : datas){
|
||||||
|
add(d)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
def run(){
|
||||||
|
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\安居客重庆小区-芝芝.xlsx",0,1)
|
||||||
|
for(def data : datas){
|
||||||
|
if(data['K']=="芝芝"){
|
||||||
|
def flag = data['M']
|
||||||
|
if(flag && flag=="新增"){
|
||||||
|
add(data)
|
||||||
|
}else{
|
||||||
|
update(data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def add(def row){
|
||||||
|
def anjukeId = row['A']
|
||||||
|
def obj = findAnjukeDistrictDetail(anjukeId)
|
||||||
|
if(obj){
|
||||||
|
def type = getEstatType(obj.extend.type)
|
||||||
|
if(type){
|
||||||
|
def city = obj.base.cityName
|
||||||
|
def region = obj.base.areaName
|
||||||
|
def lat2 = null
|
||||||
|
def lng2 = null
|
||||||
|
def lat = null
|
||||||
|
def lng = null
|
||||||
|
if(obj.base.lat){
|
||||||
|
lat2 = obj.base.lat
|
||||||
|
lng2 = obj.base.lng
|
||||||
|
|
||||||
|
JSONObject convertObj = GaoDeApiUtils.convert(lng2+","+lat2, "baidu")
|
||||||
|
if(convertObj.getInt("status") == 1) {
|
||||||
|
String lo = convertObj.get("locations")
|
||||||
|
lng = Double.parseDouble(lo.split(",")[0])
|
||||||
|
lat = Double.parseDouble(lo.split(",")[1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def distName = nameFileter(obj.base.areaName, obj.base.name)
|
||||||
|
|
||||||
|
def buildTypeStr = getBuildType(obj.base.buildTypeStr)
|
||||||
|
|
||||||
|
def address = AddressUtil.formatAddress(obj.base.city, obj.base.address)
|
||||||
|
|
||||||
|
def dist = CommonDatautils.findDistrict(city, region, distName, "district")
|
||||||
|
if(!dist){
|
||||||
|
def data = new BasicDBObject();
|
||||||
|
data.put("name_cat",0)
|
||||||
|
data.put("status",0)
|
||||||
|
data.put("cat","district")
|
||||||
|
data.put("c_date", new Date())
|
||||||
|
data.put("m_date", new Date())
|
||||||
|
data.put("city", city)
|
||||||
|
data.put("region",region)
|
||||||
|
data.put("name", distName)
|
||||||
|
data.put("estate_type2",type)
|
||||||
|
if(address){
|
||||||
|
data.put("address", obj.base.address)
|
||||||
|
}
|
||||||
|
|
||||||
|
if(lat){
|
||||||
|
Long alat = lat*10000000000L
|
||||||
|
Long alng = lng*10000000000L
|
||||||
|
data.put("lat", alat)
|
||||||
|
data.put("lng", alng)
|
||||||
|
Long alat2 = lat2.toDouble()*10000000000L
|
||||||
|
Long alng2 = lng2.toDouble()*10000000000L
|
||||||
|
data.put("lat2",alat2)
|
||||||
|
data.put("lng2",alng2)
|
||||||
|
}
|
||||||
|
data.put("finish_building_date",obj.extend.completionTime)
|
||||||
|
if(obj.extend.totalHouseHoldNum && obj.extend.totalHouseHoldNum.isNumber()){
|
||||||
|
data.put("household_count",obj.extend.totalHouseHoldNum.toInteger())
|
||||||
|
}
|
||||||
|
data.put("build_area", obj.extend.totalArea)
|
||||||
|
|
||||||
|
data.put("virescence_percent", obj.extend.landscapingRatio)
|
||||||
|
data.put("build_type", buildTypeStr)
|
||||||
|
data.put("business_quarter",obj.base.tradingAreaName)
|
||||||
|
|
||||||
|
|
||||||
|
def markMap = [:]
|
||||||
|
markMap.put("data_source", "机器入库")
|
||||||
|
data.put("mark_map",markMap)
|
||||||
|
DbUtils.seaweed.save(data);
|
||||||
|
|
||||||
|
def stop = false
|
||||||
|
while (!stop){
|
||||||
|
if(data._id.toString()){
|
||||||
|
addPrice(data)
|
||||||
|
stop = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
println "【物业类型问题】 ${obj._id.toString()} "+obj.extend.type
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
println anjukeId+" 没找到"
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
def update(def row){
|
||||||
|
def commId = row['L']
|
||||||
|
if(!commId){
|
||||||
|
commId = row['G']
|
||||||
|
}
|
||||||
|
|
||||||
|
def dist = CommonDatautils.queryDistrictById(commId)
|
||||||
|
def anjukeId = row['A']
|
||||||
|
def obj = findAnjukeDistrictDetail(anjukeId)
|
||||||
|
if(obj && dist){
|
||||||
|
def udata = new BasicDBObject()
|
||||||
|
|
||||||
|
def address = AddressUtil.formatAddress(obj.base.cityName, obj.base.address)
|
||||||
|
if(address){
|
||||||
|
if(!dist.address){
|
||||||
|
if(obj.base.address){
|
||||||
|
udata.put("address",obj.base.address)
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
if(dist.address.contains("号")){
|
||||||
|
if(!dist.address.contains(obj.base.address) && !obj.base.address.contains(dist.address)){
|
||||||
|
def addr_alias = dist.addr_alias
|
||||||
|
if(!addr_alias){
|
||||||
|
addr_alias = []
|
||||||
|
}
|
||||||
|
if(!addr_alias.contains(obj.base.address) && obj.base.address.contains("号")){
|
||||||
|
addr_alias << obj.base.address
|
||||||
|
}
|
||||||
|
if(addr_alias!=dist.addr_alias && addr_alias.size()>0){
|
||||||
|
udata.put("addr_alias", addr_alias)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
udata.put("address",obj.base.address)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def distName = nameFileter(obj.base.areaName, obj.base.name)
|
||||||
|
if(!distName.contains("号")){
|
||||||
|
def alias = dist.alias
|
||||||
|
if(!alias){
|
||||||
|
alias = []
|
||||||
|
}
|
||||||
|
if(alias.contains(distName)){
|
||||||
|
alias << distName
|
||||||
|
}
|
||||||
|
if(alias.size()>0 && alias!=dist.alias){
|
||||||
|
udata.put("alias", alias)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if(!dist.build_type){
|
||||||
|
def buildTypeStr = getBuildType(obj.base.buildTypeStr)
|
||||||
|
if(buildTypeStr){
|
||||||
|
udata.put("build_type",buildTypeStr)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
if(!dist.household_count){
|
||||||
|
if(obj.extend.totalHouseHoldNum && obj.extend.totalHouseHoldNum.isNumber() && obj.extend.totalHouseHoldNum.toInteger()>0){
|
||||||
|
udata.put("household_count",obj.extend.totalHouseHoldNum.toInteger())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(!dist.finish_building_date){
|
||||||
|
if(obj.extend.completionTime){
|
||||||
|
udata.put("finish_building_date",obj.extend.completionTime)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if(!dist.virescence_percent){
|
||||||
|
if(obj.extend.landscapingRatio){
|
||||||
|
udata.put("virescence_percent", obj.extend.landscapingRatio)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(!dist.business_quarter){
|
||||||
|
if(obj.base.tradingAreaName){
|
||||||
|
udata.put("business_quarter",obj.base.tradingAreaName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
def lat2 = null
|
||||||
|
def lng2 = null
|
||||||
|
if(dist.lat2){
|
||||||
|
lat2 = dist.lat2/10000000000.0
|
||||||
|
lng2 = dist.lng2/10000000000.0
|
||||||
|
}
|
||||||
|
if(dist.lat && !lat2){
|
||||||
|
// 高德转百度
|
||||||
|
def baidu = BaiDuApiUtils.geoconv(dist.lng/10000000000.0, dist.lat/10000000000.0, 3)
|
||||||
|
lng2 = baidu.x
|
||||||
|
lat2 = baidu.y
|
||||||
|
|
||||||
|
Long dLat2 = lat2 * 10000000000L
|
||||||
|
Long dLng2 = lng2 * 10000000000L
|
||||||
|
udata.put("lat2", dLat2)
|
||||||
|
udata.put("lng2", dLng2)
|
||||||
|
}
|
||||||
|
|
||||||
|
if(obj.base.lat) {
|
||||||
|
def alat2 = obj.base.lat.toDouble()
|
||||||
|
def alng2 = obj.base.lng.toDouble()
|
||||||
|
def distance = GeoUtils.calculateDistance(lng2, lat2, alng2, alat2);
|
||||||
|
if(distance>1000){
|
||||||
|
println "........${obj.base.name}..............${dist.name}..........距离相差${distance}......................"
|
||||||
|
|
||||||
|
def lng = null
|
||||||
|
def lat = null
|
||||||
|
JSONObject convertObj = GaoDeApiUtils.convert(obj.base.lng+","+obj.base.lat, "baidu")
|
||||||
|
if(convertObj.getInt("status") == 1) {
|
||||||
|
String lo = convertObj.get("locations")
|
||||||
|
lng = Double.parseDouble(lo.split(",")[0])
|
||||||
|
lat = Double.parseDouble(lo.split(",")[1])
|
||||||
|
}
|
||||||
|
|
||||||
|
Long blat2 = alat2 *10000000000L
|
||||||
|
Long blng2 = alng2 *10000000000L
|
||||||
|
Long gLat2 = lat *10000000000L
|
||||||
|
Long gLng2 = lng *10000000000L
|
||||||
|
|
||||||
|
udata.put("lat2", blat2)
|
||||||
|
udata.put("lng2", blng2)
|
||||||
|
udata.put("lat", gLat2)
|
||||||
|
udata.put("lng", gLng2)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if(udata){
|
||||||
|
udata.put("m_date", new Date())
|
||||||
|
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||||
|
println "[修改] ${dist.name}"+ udata
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def findAnjukeDistrictDetail(anjukeId){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("_id",new ObjectId(anjukeId))
|
||||||
|
return detail.findOne(q)
|
||||||
|
}
|
||||||
|
|
||||||
|
def nameFileter(def region, def name){
|
||||||
|
name = name.replace("·","").replace(".","").replace("|","")
|
||||||
|
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
|
||||||
|
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
|
||||||
|
name = name.replace("(${region})","").replace("(","").replace(")","")
|
||||||
|
return name;
|
||||||
|
}
|
||||||
|
|
||||||
|
def getEstatType(type){
|
||||||
|
if(type){
|
||||||
|
if(type=="公寓住宅"){
|
||||||
|
type = "普通住宅"
|
||||||
|
}else if(type.contains("商住")){
|
||||||
|
type = "商住"
|
||||||
|
}else if(type.contains("别墅")){
|
||||||
|
type = "别墅"
|
||||||
|
}else{
|
||||||
|
type = null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return type
|
||||||
|
}
|
||||||
|
|
||||||
|
def getBuildType(def buildType){
|
||||||
|
def types = ["多层","小高层","高层","超高层","花园洋房","老洋房","四合院","独栋","双拼","联排","叠加","老式别墅洋房"]
|
||||||
|
def list = []
|
||||||
|
if(buildType){
|
||||||
|
def arrs = buildType.split("\\|")
|
||||||
|
for(def s : arrs){
|
||||||
|
if(types.contains(s)){
|
||||||
|
list << s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(list){
|
||||||
|
return list.join(",")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
def getBuildType2(def buildType){
|
||||||
|
def types = ["多层","小高层","高层","超高层","花园洋房","老洋房","四合院","独栋","双拼","联排","叠加","老式别墅洋房"]
|
||||||
|
def list = []
|
||||||
|
if(buildType){
|
||||||
|
def arrs = buildType.split(",| ")
|
||||||
|
for(def s : arrs){
|
||||||
|
if(types.contains(s)){
|
||||||
|
list << s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(list){
|
||||||
|
return list.join(",")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
def addPrice(dist){
|
||||||
|
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-04-01")
|
||||||
|
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-05-25")
|
||||||
|
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
|
||||||
|
|
||||||
|
for(def weekend : weekends){
|
||||||
|
CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+4
-4
@@ -122,7 +122,7 @@ public class BeiKeCrawlByChrome {
|
|||||||
def durl = e.select(".title a").attr("href")
|
def durl = e.select(".title a").attr("href")
|
||||||
def dealDate = e.select(".dealDate").text()
|
def dealDate = e.select(".dealDate").text()
|
||||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||||
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
|
def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd")
|
||||||
if(dealDate.getTime()>=endDate.getTime()){
|
if(dealDate.getTime()>=endDate.getTime()){
|
||||||
detailUrls << durl
|
detailUrls << durl
|
||||||
}else{
|
}else{
|
||||||
@@ -153,7 +153,7 @@ public class BeiKeCrawlByChrome {
|
|||||||
def durl = e.select(".title a").attr("href")
|
def durl = e.select(".title a").attr("href")
|
||||||
def dealDate = e.select(".dealDate").text()
|
def dealDate = e.select(".dealDate").text()
|
||||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||||
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
|
def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd")
|
||||||
if(dealDate && dealDate.getTime()>=endDate.getTime()){
|
if(dealDate && dealDate.getTime()>=endDate.getTime()){
|
||||||
detailUrls << durl
|
detailUrls << durl
|
||||||
parse(city, e)
|
parse(city, e)
|
||||||
@@ -358,13 +358,13 @@ public class BeiKeCrawlByChrome {
|
|||||||
if(cityEles){
|
if(cityEles){
|
||||||
for(def e : cityEles){
|
for(def e : cityEles){
|
||||||
def href = e.attr("href")
|
def href = e.attr("href")
|
||||||
if(e.text().contains("ÖØÇì")){
|
// if(e.text().contains("¸£ÖÝ")){
|
||||||
def cityMap = [:]
|
def cityMap = [:]
|
||||||
cityMap.city = e.text()
|
cityMap.city = e.text()
|
||||||
cityMap.url = "https:"+href
|
cityMap.url = "https:"+href
|
||||||
list << cityMap
|
list << cityMap
|
||||||
//cityMap.put(e.text(), "https:"+href)
|
//cityMap.put(e.text(), "https:"+href)
|
||||||
}
|
// }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return list
|
return list
|
||||||
|
|||||||
+82
-75
@@ -29,7 +29,7 @@ class JdPaimaiCrawl {
|
|||||||
task.run();
|
task.run();
|
||||||
}
|
}
|
||||||
def run(){
|
def run(){
|
||||||
def existsIds = findAllIds()
|
def existsIds = []//findAllIds()
|
||||||
println "ÒÑ´æÔÚ:"+existsIds.size()
|
println "ÒÑ´æÔÚ:"+existsIds.size()
|
||||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||||
System.setProperty("http.nonProxyHosts", "localhost");
|
System.setProperty("http.nonProxyHosts", "localhost");
|
||||||
@@ -43,78 +43,80 @@ class JdPaimaiCrawl {
|
|||||||
|
|
||||||
def getList(driver){
|
def getList(driver){
|
||||||
def list = []
|
def list = []
|
||||||
for(def i=1;i<=32;i++){
|
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId=4&_=1564047626268"
|
||||||
if(i==8){
|
// for(def i=1;i<=32;i++){
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=560&_=1564047626268"
|
// if(i==8){
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=573&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=560&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=579&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=573&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=584&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=579&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=589&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=584&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=593&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=589&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=598&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=593&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=604&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=598&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=609&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=604&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=613&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=609&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=617&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=613&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=621&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=617&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=632&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=621&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=6858&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=632&_=1564047626268"
|
||||||
}else if(i==5){
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=6858&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=142&_=1564047626268"
|
// }else if(i==5){
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=148&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=142&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=164&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=148&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=199&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=164&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=224&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=199&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=239&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=224&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=248&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=239&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=258&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=248&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=264&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=258&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=274&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=264&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=275&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=274&_=1564047626268"
|
||||||
}else if(i==17){
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=275&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1432&_=1564047626268"
|
// }else if(i==17){
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1441&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1432&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1458&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1441&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1466&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1458&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1475&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1466&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1477&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1475&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1479&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1477&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=3154&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1479&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1381&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=3154&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1387&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1381&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1396&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1387&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1405&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1396&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1413&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1405&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1421&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1413&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2922&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1421&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2980&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2922&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2983&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2980&_=1564047626268"
|
||||||
}else if(i==19){
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2983&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1601&_=1564047626268"
|
// }else if(i==19){
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1601&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1611&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1617&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1611&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1627&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1617&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1634&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1627&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1709&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1634&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1643&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1709&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1650&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1643&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1655&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1650&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1655&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1666&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1672&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1666&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1677&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1672&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1684&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1677&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1690&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1684&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1698&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1690&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1704&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1698&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1705&_=1564047626268"
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1704&_=1564047626268"
|
||||||
}else{
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1705&_=1564047626268"
|
||||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&_=1564047626268"
|
// }else{
|
||||||
}
|
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&_=1564047626268"
|
||||||
}
|
// }
|
||||||
|
|
||||||
|
// }
|
||||||
def links = []
|
def links = []
|
||||||
for(def url : list){
|
for(def url : list){
|
||||||
driver.get(url)
|
driver.get(url)
|
||||||
@@ -430,10 +432,10 @@ class JdPaimaiCrawl {
|
|||||||
|
|
||||||
def s1 = JQueryUtils.selectText(["li:contains(¾ºÂò¹«¸æ) .floor-content"], doc)
|
def s1 = JQueryUtils.selectText(["li:contains(¾ºÂò¹«¸æ) .floor-content"], doc)
|
||||||
if(s1) {
|
if(s1) {
|
||||||
def start = s1.indexOf("·¨Ôº½«ÓÚ")
|
def start = s1.indexOf("将于")
|
||||||
def end = s1.indexOf("Ö¹")
|
def end = s1.indexOf("Ö¹")
|
||||||
if(start>-1 && end>-1 && end>start){
|
if(start>-1 && end>-1 && end>start){
|
||||||
s1 = s1.substring(s1.indexOf("·¨Ôº½«ÓÚ"), s1.indexOf("Ö¹"))
|
s1 = s1.substring(s1.indexOf("将于"), s1.indexOf("止"))
|
||||||
}
|
}
|
||||||
def dateRangs = DataUtil.getDateRange(s1)
|
def dateRangs = DataUtil.getDateRange(s1)
|
||||||
if(dateRangs && dateRangs.size()==2) {
|
if(dateRangs && dateRangs.size()==2) {
|
||||||
@@ -476,6 +478,11 @@ class JdPaimaiCrawl {
|
|||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if(!result.startDate){
|
||||||
|
def sTime = JQueryUtils.selectText(".startTime", doc).replace("开拍时间:","")
|
||||||
|
result.startDate = DateUtils.parseFromTightByPattern(sTime, "yyyy年MM月dd日")
|
||||||
|
}
|
||||||
if(result.finishDate){
|
if(result.finishDate){
|
||||||
result.finishDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.finishDate, "2000-01-01")
|
result.finishDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.finishDate, "2000-01-01")
|
||||||
}
|
}
|
||||||
|
|||||||
+27
-7
@@ -34,10 +34,30 @@ public class LianjiaCrawlByChrome {
|
|||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new LianjiaCrawlByChrome();
|
def task = new LianjiaCrawlByChrome();
|
||||||
task.run()
|
// task.run()
|
||||||
// task.run1()
|
// task.run1()
|
||||||
task.addDealRegion();
|
// task.addDealRegion();
|
||||||
task.standardRegion()
|
// task.standardRegion()
|
||||||
|
task.updadteDealPriceField()
|
||||||
|
}
|
||||||
|
|
||||||
|
def updadteDealPriceField(){
|
||||||
|
def citys = CommonDatautils.findAllCity()
|
||||||
|
def index = 0
|
||||||
|
for(def city : citys){
|
||||||
|
index ++
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("city", city)
|
||||||
|
q.put("hall", "0")
|
||||||
|
def count = DEAL_PRICE_COLL.count(q)
|
||||||
|
println "[${index}] "+city +" "+count
|
||||||
|
if(count && count>0){
|
||||||
|
def udata = new BasicDBObject()
|
||||||
|
udata.put("hall", 0)
|
||||||
|
DEAL_PRICE_COLL.update(q, new BasicDBObject('$set', udata), false, true);
|
||||||
|
println "[ÐÞ¸Ä]"+city +" "+ count
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
def run1(){
|
def run1(){
|
||||||
@@ -187,7 +207,7 @@ public class LianjiaCrawlByChrome {
|
|||||||
def durl = e.select(".title a").attr("href")
|
def durl = e.select(".title a").attr("href")
|
||||||
def dealDate = e.select(".dealDate").text()
|
def dealDate = e.select(".dealDate").text()
|
||||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||||
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
|
def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd")
|
||||||
if(dealDate.getTime()>=endDate.getTime()){
|
if(dealDate.getTime()>=endDate.getTime()){
|
||||||
detailUrls << durl
|
detailUrls << durl
|
||||||
}else{
|
}else{
|
||||||
@@ -218,7 +238,7 @@ public class LianjiaCrawlByChrome {
|
|||||||
def durl = e.select(".title a").attr("href")
|
def durl = e.select(".title a").attr("href")
|
||||||
def dealDate = e.select(".dealDate").text()
|
def dealDate = e.select(".dealDate").text()
|
||||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||||
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
|
def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd")
|
||||||
if(dealDate.getTime()>=endDate.getTime()){
|
if(dealDate.getTime()>=endDate.getTime()){
|
||||||
detailUrls << durl
|
detailUrls << durl
|
||||||
parse(city, e)
|
parse(city, e)
|
||||||
@@ -420,11 +440,11 @@ public class LianjiaCrawlByChrome {
|
|||||||
def cityDoc = Jsoup.parse(driver.getPageSource());
|
def cityDoc = Jsoup.parse(driver.getPageSource());
|
||||||
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
|
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
|
||||||
def cityMap = [:]
|
def cityMap = [:]
|
||||||
def citys= ["³¤´º","¼ªÁÖ"]
|
def citys= ["¸£ÖÝ"]
|
||||||
if(cityEles){
|
if(cityEles){
|
||||||
for(def e : cityEles) {
|
for(def e : cityEles) {
|
||||||
def href = e.attr("href")
|
def href = e.attr("href")
|
||||||
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
|
if (!href.contains("fang.lianjia")){
|
||||||
cityMap.put(e.text(), href)
|
cityMap.put(e.text(), href)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+15
-4
@@ -30,8 +30,8 @@ class PaimaiTest {
|
|||||||
def task = new PaimaiTest();
|
def task = new PaimaiTest();
|
||||||
// task.run();
|
// task.run();
|
||||||
// task.stat()
|
// task.stat()
|
||||||
// task.updatePaimai()
|
task.updatePaimai()
|
||||||
task.updateFieldPaimai()
|
// task.updateFieldPaimai()
|
||||||
}
|
}
|
||||||
|
|
||||||
def updateFieldPaimai(){
|
def updateFieldPaimai(){
|
||||||
@@ -42,8 +42,8 @@ class PaimaiTest {
|
|||||||
for(def obj : dbcursor) {
|
for(def obj : dbcursor) {
|
||||||
index++
|
index++
|
||||||
println index
|
println index
|
||||||
if(obj.depositPrice){
|
if(obj.address && obj.address.contains("可贷款")){
|
||||||
obj.put("depositPrice", obj.depositPrice.toInteger())
|
obj.put("address", obj.address.substring(0,obj.address.indexOf("可贷款")))
|
||||||
paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj))
|
paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -51,11 +51,22 @@ class PaimaiTest {
|
|||||||
|
|
||||||
def updatePaimai(){
|
def updatePaimai(){
|
||||||
def q = new BasicDBObject()
|
def q = new BasicDBObject()
|
||||||
|
q.put("url", Pattern.compile(".*jd.com.*"))
|
||||||
def dbcursor = paimaiResultColl.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
def dbcursor = paimaiResultColl.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||||
def index =0
|
def index =0
|
||||||
for(def d : dbcursor){
|
for(def d : dbcursor){
|
||||||
index++
|
index++
|
||||||
println index
|
println index
|
||||||
|
if(!d.url.contains("jd.com")){
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if(!d.proince.contains("重庆")){
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
println d.url
|
||||||
|
|
||||||
def query = new BasicDBObject()
|
def query = new BasicDBObject()
|
||||||
query.put("url", d.url)
|
query.put("url", d.url)
|
||||||
def dbo = paimai43_COLL.findOne(query);
|
def dbo = paimai43_COLL.findOne(query);
|
||||||
|
|||||||
+34
-1
@@ -42,8 +42,41 @@ class CqfdcMatch {
|
|||||||
// task.export2()
|
// task.export2()
|
||||||
// task.matchLoudong()
|
// task.matchLoudong()
|
||||||
|
|
||||||
task.addLoudong()
|
// task.addLoudong()
|
||||||
// task.addFdcLoudong()
|
// task.addFdcLoudong()
|
||||||
|
task.updateHouseLoudong()
|
||||||
|
}
|
||||||
|
|
||||||
|
def updateHouseLoudong(){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("city", "ÖØÇì")
|
||||||
|
|
||||||
|
def keys = new BasicDBObject()
|
||||||
|
keys.put("room_num", 1)
|
||||||
|
keys.put("house_floor", 1)
|
||||||
|
def dbcursor = HOUSE_LOUDONG_COLL.find(q).batchSize(5000)
|
||||||
|
def index = 0
|
||||||
|
for(def d : dbcursor){
|
||||||
|
index++
|
||||||
|
def room_num = d.room_num
|
||||||
|
def house_floor = d.house_floor
|
||||||
|
try{
|
||||||
|
if(room_num && house_floor && room_num.isNumber()){
|
||||||
|
if(house_floor.toInteger()>0){
|
||||||
|
if(!room_num.toString().startsWith(house_floor.toString()) && room_num.toInteger()<=10){
|
||||||
|
def newValue = house_floor +"-"+room_num
|
||||||
|
println "[${index}] "+room_num +" "+house_floor+" "+newValue
|
||||||
|
def udata = new BasicDBObject()
|
||||||
|
udata.put("room_num",newValue)
|
||||||
|
HOUSE_LOUDONG_COLL.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata), false, false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}catch(Exception e){
|
||||||
|
println d._id.toString()
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
}
|
}
|
||||||
def export(){
|
def export(){
|
||||||
def q = new BasicDBObject()
|
def q = new BasicDBObject()
|
||||||
|
|||||||
+3
-3
@@ -40,11 +40,11 @@ class DataStat {
|
|||||||
|
|
||||||
|
|
||||||
def run(){
|
def run(){
|
||||||
def source = "58ͬ³Ç"//
|
def source = "±´¿ÇÕÒ·¿"//
|
||||||
def citys = findCrawlCity(source, 202202);
|
def citys = findCrawlCity(source, 202202);
|
||||||
for(def city : citys){
|
for(def city : citys){
|
||||||
def count1 = findCount(source, city, 202202);
|
def count1 = findCount(source, city, 202203);
|
||||||
def count2 = findCount(source, city, 202203);
|
def count2 = findCount(source, city, 202204);
|
||||||
println city +" "+count1+" "+count2
|
println city +" "+count1+" "+count2
|
||||||
// println count1
|
// println count1
|
||||||
// println "--- ----"
|
// println "--- ----"
|
||||||
|
|||||||
+4
-3
@@ -192,7 +192,8 @@ class Esf58Crawl extends BaseBatch{
|
|||||||
void process(Object args) {
|
void process(Object args) {
|
||||||
//addContent("¸£ÖÝ","https://fz.58.com/xiaoqu/", 70);
|
//addContent("¸£ÖÝ","https://fz.58.com/xiaoqu/", 70);
|
||||||
|
|
||||||
// def citys = ["北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏"]
|
def citys = ["上海","北京","广州","天津","武汉","南京","青岛","合肥","西安","沈阳","宁波","佛山","潍坊","济南","大连","兰州","石家庄","长春","温州","乌鲁木齐","太原","惠州","嘉兴","南通","南昌","烟台","临沂","呼和浩特","厦门","海口","常州","中山","绍兴","淄博","洛阳","珠海","金华","威海","德阳","唐山","泰安","襄阳","台州","镇江","银川","江门","荆州","衡阳","滨州","张家口","赣州","商丘","安庆","湖州","枣庄","聊城","阜阳","芜湖","吉林","遂宁","鞍山","东营","安阳","咸阳","蚌埠","九江","临汾","大同","六安","淮南","宿州","三亚","玉溪","六盘水","揭阳","汕尾","恩施","盘锦","伊犁","玉林","随州","抚州","景德镇","四平","丽水","安顺","毕节","天水","阜新","通化","攀枝花","吕梁","阿克苏","新余","松原","楚雄","鹤壁","忻州","本溪","拉萨","三门峡","黔东南","兴安盟","雅安","仙桃","乌海","贵港","安康","白城","平凉","庆阳","延安","潮州","白银","黔西南","丽江","吴忠","防城港","喀什","哈尔滨","钦州","鸡西","石嘴山","哈密","萍乡","鹰潭","武威","保山","铜仁","普洱","酒泉","宜昌","河池","朔州","百色","鹤岗","张家界","湘西","张掖","定西","昭通","济源","中卫","黑河","克拉玛依","铜川","辽源","云浮","博尔塔拉","固原","白山","广元","贺州","潜江","天门","伊春","石河子","临沧","榆林","临夏","来宾","成都","双鸭山","梧州","嘉峪关","海东","昌吉"]
|
||||||
|
|
||||||
// for(def city : citys){
|
// for(def city : citys){
|
||||||
// def str = cityList.get(city)
|
// def str = cityList.get(city)
|
||||||
// for(def i=0;i<1;i++){
|
// for(def i=0;i<1;i++){
|
||||||
@@ -209,7 +210,7 @@ class Esf58Crawl extends BaseBatch{
|
|||||||
|
|
||||||
cityList.each{city, str->
|
cityList.each{city, str->
|
||||||
for(def i=0;i<1;i++){
|
for(def i=0;i<1;i++){
|
||||||
// if(citys.contains(city)){
|
if(citys.contains(city)){
|
||||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
|
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
|
||||||
if(districts){
|
if(districts){
|
||||||
def arrs = str.split("\\|")
|
def arrs = str.split("\\|")
|
||||||
@@ -219,7 +220,7 @@ class Esf58Crawl extends BaseBatch{
|
|||||||
addCrawlKeys(city);
|
addCrawlKeys(city);
|
||||||
addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
|
addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
|
||||||
}
|
}
|
||||||
// }
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+2
-2
@@ -18,7 +18,7 @@ class SynchDistrictPrice {
|
|||||||
// def sourceMap = ["fang.com":"搜房","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
|
// def sourceMap = ["fang.com":"搜房","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
|
||||||
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"]
|
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"]
|
||||||
//"fang.com":"搜房",
|
//"fang.com":"搜房",
|
||||||
def sourceMap = ["fang.com":"搜房","ke.com":"贝壳找房","5i5j.com":"我爱我家"]
|
def sourceMap = ["ke.com":"贝壳找房","5i5j.com":"我爱我家"]
|
||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new SynchDistrictPrice()
|
def task = new SynchDistrictPrice()
|
||||||
task.run()
|
task.run()
|
||||||
@@ -26,7 +26,7 @@ class SynchDistrictPrice {
|
|||||||
|
|
||||||
|
|
||||||
def run(){
|
def run(){
|
||||||
def citys = CommonUtils.findAllCity()
|
def citys = ["重庆"]//CommonUtils.findAllCity()
|
||||||
|
|
||||||
sourceMap.each { k, v ->
|
sourceMap.each { k, v ->
|
||||||
for(def city : citys){
|
for(def city : citys){
|
||||||
|
|||||||
+11
-10
@@ -236,10 +236,10 @@ public class OpenApiClient {
|
|||||||
params.put("houseNumber", houseNum);
|
params.put("houseNumber", houseNum);
|
||||||
params.put("unitNumber", unitNum);
|
params.put("unitNumber", unitNum);
|
||||||
params.put("token", TOKEN);
|
params.put("token", TOKEN);
|
||||||
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/property/searchLoudong", params, "utf-8")
|
String content = new HttpRequest().get("http://open.fangjia.com/property/searchLoudong", params, "utf-8")
|
||||||
JSONObject result = JSONObject.fromObject(content);
|
JSONObject result = JSONObject.fromObject(content);
|
||||||
while (result == null) {
|
while (result == null) {
|
||||||
result = HttpRequestUtil.getJsonContent("http://localhost:8080/fangjia-open/property/searchLoudong", params, "utf-8")
|
result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/property/searchLoudong", params, "utf-8")
|
||||||
println("${id} ${houseNum} 程序调用异常")
|
println("${id} ${houseNum} 程序调用异常")
|
||||||
Thread.sleep(6000)
|
Thread.sleep(6000)
|
||||||
}
|
}
|
||||||
@@ -266,10 +266,10 @@ public class OpenApiClient {
|
|||||||
params.put("unitNumber", unitNum);
|
params.put("unitNumber", unitNum);
|
||||||
params.put("roomNumber", roomNum);
|
params.put("roomNumber", roomNum);
|
||||||
params.put("token", TOKEN);
|
params.put("token", TOKEN);
|
||||||
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/property/searchHouseLoudong", params, "utf-8")
|
String content = new HttpRequest().get("http://open.fangjia.com/property/searchHouseLoudong", params, "utf-8")
|
||||||
JSONObject result = JSONObject.fromObject(content);
|
JSONObject result = JSONObject.fromObject(content);
|
||||||
while (result == null) {
|
while (result == null) {
|
||||||
result = HttpRequestUtil.getJsonContent("http://localhost:8080/fangjia-open/property/searchHouseLoudong", params, "utf-8")
|
result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/property/searchHouseLoudong", params, "utf-8")
|
||||||
println("${id} ${houseNum} 程序调用异常")
|
println("${id} ${houseNum} 程序调用异常")
|
||||||
Thread.sleep(6000)
|
Thread.sleep(6000)
|
||||||
}
|
}
|
||||||
@@ -687,7 +687,7 @@ public class OpenApiClient {
|
|||||||
params.put("region", region)
|
params.put("region", region)
|
||||||
params.put("name", name)
|
params.put("name", name)
|
||||||
try {
|
try {
|
||||||
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/district/facilitys", params, "utf-8")
|
String content = new HttpRequest().get("http://open.fangjia.com/district/facilitys", params, "utf-8")
|
||||||
JSONObject result = JSONObject.fromObject(content);
|
JSONObject result = JSONObject.fromObject(content);
|
||||||
|
|
||||||
return result
|
return result
|
||||||
@@ -697,13 +697,14 @@ public class OpenApiClient {
|
|||||||
return null
|
return null
|
||||||
}
|
}
|
||||||
|
|
||||||
static JSONObject searchPaimais(String city, String region, String name, limit) {
|
static JSONObject searchPaimais(String city, String region, String name, String startDate,limit) {
|
||||||
Map<String, String> params = new HashMap<String, String>();
|
Map<String, String> params = new HashMap<String, String>();
|
||||||
params.put("token", TOKEN);
|
params.put("token", TOKEN);
|
||||||
params.put("city", city)
|
params.put("city", city)
|
||||||
params.put("district", region)
|
params.put("district", region)
|
||||||
params.put("name", name)
|
params.put("name", name)
|
||||||
params.put("limit", "10")
|
params.put("startDate", startDate)
|
||||||
|
params.put("limit", "100")
|
||||||
try {
|
try {
|
||||||
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/auction/searchPaimais", params, "utf-8")
|
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/auction/searchPaimais", params, "utf-8")
|
||||||
JSONObject result = JSONObject.fromObject(content);
|
JSONObject result = JSONObject.fromObject(content);
|
||||||
@@ -722,7 +723,7 @@ public class OpenApiClient {
|
|||||||
params.put("startDate", startDate)
|
params.put("startDate", startDate)
|
||||||
params.put("limit", limit.toString())
|
params.put("limit", limit.toString())
|
||||||
try {
|
try {
|
||||||
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/dealPrice/searchList", params, "utf-8")
|
String content = new HttpRequest().get("http://open.fangjia.com/dealPrice/searchList", params, "utf-8")
|
||||||
JSONObject result = JSONObject.fromObject(content);
|
JSONObject result = JSONObject.fromObject(content);
|
||||||
return result
|
return result
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
@@ -738,7 +739,7 @@ public class OpenApiClient {
|
|||||||
params.put("startTime", startDate)
|
params.put("startTime", startDate)
|
||||||
params.put("pageSize", limit.toString())
|
params.put("pageSize", limit.toString())
|
||||||
try {
|
try {
|
||||||
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/house/list", params, "utf-8")
|
String content = new HttpRequest().get("http://open.fangjia.com/house/list", params, "utf-8")
|
||||||
JSONObject result = JSONObject.fromObject(content);
|
JSONObject result = JSONObject.fromObject(content);
|
||||||
return result
|
return result
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
@@ -755,7 +756,7 @@ public class OpenApiClient {
|
|||||||
params.put("startDate", startDate)
|
params.put("startDate", startDate)
|
||||||
params.put("limit", limit.toString())
|
params.put("limit", limit.toString())
|
||||||
try {
|
try {
|
||||||
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/evaluation/bsList", params, "utf-8")
|
String content = new HttpRequest().get("http://open.fangjia.com/evaluation/bsList", params, "utf-8")
|
||||||
JSONObject result = JSONObject.fromObject(content);
|
JSONObject result = JSONObject.fromObject(content);
|
||||||
return result
|
return result
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
|
|||||||
+9
@@ -0,0 +1,9 @@
|
|||||||
|
package com.fangjia.service.open;
|
||||||
|
|
||||||
|
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
|
||||||
|
import com.fangjia.dal.mongo.open.model.CommunityPoi;
|
||||||
|
|
||||||
|
public interface CommunityBasePriceService {
|
||||||
|
|
||||||
|
CommunityBasePrice getById(String id);
|
||||||
|
}
|
||||||
+22
@@ -0,0 +1,22 @@
|
|||||||
|
package com.fangjia.service.open.impl;
|
||||||
|
|
||||||
|
import com.fangjia.dal.mongo.open.dao.CommunityBasePriceDao;
|
||||||
|
import com.fangjia.dal.mongo.open.dao.CommunityPoiDao;
|
||||||
|
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
|
||||||
|
import com.fangjia.dal.mongo.open.model.CommunityPoi;
|
||||||
|
import com.fangjia.service.open.CommunityBasePriceService;
|
||||||
|
import com.fangjia.service.open.CommunityPoiService;
|
||||||
|
|
||||||
|
public class CommunityBasePriceServiceImpl implements CommunityBasePriceService {
|
||||||
|
|
||||||
|
private CommunityBasePriceDao communityBasePriceDao;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public CommunityBasePrice getById(String id) {
|
||||||
|
return communityBasePriceDao.findById(id);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setCommunityBasePriceDao(CommunityBasePriceDao communityBasePriceDao) {
|
||||||
|
this.communityBasePriceDao = communityBasePriceDao;
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user