Merge branch 'master' of http://gitlab.fangjia.com/dev0/main into open_bohai

This commit is contained in:
2022-06-15 15:38:39 +08:00
95 changed files with 7887 additions and 708 deletions
@@ -158,37 +158,36 @@
<a class="partner13"></a>
<#-- <a class="partner14"></a>-->
<a class="partner15"></a>
<a class="partner16" style="margin-right:0;"></a>
<a class="partner16"></a>
<a class="partner21"></a>
<a class="partner22"></a>
<a class="partner23"></a>
<a class="partner24"></a>
<a class="partner25"></a>
<a class="partner26" style="margin-right:0;"></a>
<a class="partner26"></a>
<a class="partner31"></a>
<a class="partner32"></a>
<a class="partner33"></a>
<a class="partner34"></a>
<a class="partner35"></a>
<a class="partner36" style="margin-right:0;"></a>
<a class="partner36"></a>
<a class="partner41"></a>
<a class="partner42"></a>
<a class="partner43"></a>
<a class="partner44"></a>
<a class="partner45"></a>
<a class="partner46" style="margin-right:0;"></a>
<a class="partner46"></a>
<a class="partner51"></a>
<a class="partner52"></a>
<a class="partner53"></a>
<a class="partner54"></a>
<a class="partner55"></a>
<a class="partner56" style="margin-right:0;"></a>
<a class="partner56"></a>
<a class="partner61"></a>
<a class="partner62"></a>
<a class="partner63"></a>
<a class="partner65"></a>
<a class="partner66"></a>
<a class="partner71" style="margin-right:0;"></a>
<a class="partner72"></a>
<a class="partner73"></a>
</div>
@@ -0,0 +1,9 @@
package com.fangjia.dal.mongo.open.dao;
import com.fangjia.dal.MongoDao;
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
import com.fangjia.dal.mongo.open.model.CommunityPoi;
public interface CommunityBasePriceDao extends MongoDao<CommunityBasePrice> {
}
@@ -0,0 +1,9 @@
package com.fangjia.dal.mongo.open.dao;
import com.fangjia.dal.MongoDao;
import com.fangjia.dal.mongo.model.CountDistrictStatBycity;
import com.fangjia.dal.mongo.open.model.CommunityPoi;
public interface CommunityPoiDao extends MongoDao<CommunityPoi> {
}
@@ -0,0 +1,10 @@
package com.fangjia.dal.mongo.open.dao.impl;
import com.fangjia.dal.MongoDaoImpl;
import com.fangjia.dal.mongo.open.dao.CommunityBasePriceDao;
import com.fangjia.dal.mongo.open.dao.CommunityPoiDao;
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
import com.fangjia.dal.mongo.open.model.CommunityPoi;
public class CommunityBasePriceDaoImpl extends MongoDaoImpl<CommunityBasePrice> implements CommunityBasePriceDao {
}
@@ -0,0 +1,10 @@
package com.fangjia.dal.mongo.open.dao.impl;
import com.fangjia.dal.MongoDaoImpl;
import com.fangjia.dal.mongo.open.dao.ApiVisitCountDao;
import com.fangjia.dal.mongo.open.dao.CommunityPoiDao;
import com.fangjia.dal.mongo.open.model.CommunityPoi;
import com.fangjia.dal.mongo.tool.model.ApiVisitCount;
public class CommunityPoiDaoImpl extends MongoDaoImpl<CommunityPoi> implements CommunityPoiDao {
}
@@ -0,0 +1,74 @@
package com.fangjia.dal.mongo.open.model;
import com.fangjia.dal.MongoEntity;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.Table;
/**
* Ð¡Çø½¨ÐÐÖÃÐŶÈÐÅÏ¢
*
* @author shenjian@fangjia.com
*/
@Entity
@Table(name = "business.community_base_price_gj")
public class CommunityBasePrice implements MongoEntity {
private static final long serialVersionUID = -8010010773207622878L;
private String id;
private String city;
private String region;
private String name;
private Double credit;
@Id
@Column(name = "_id")
@Override
public String getId() {
return id;
}
@Override
public void setId(String id) {
this.id = id;
}
@Column(name = "city")
public String getCity() {
return city;
}
public void setCity(String city) {
this.city = city;
}
@Column(name = "region")
public String getRegion() {
return region;
}
public void setRegion(String region) {
this.region = region;
}
@Column(name = "name")
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
@Column(name = "credit")
public Double getCredit() {
return credit;
}
public void setCredit(Double credit) {
this.credit = credit;
}
}
@@ -0,0 +1,135 @@
package com.fangjia.dal.mongo.open.model;
import com.fangjia.dal.MongoEntity;
import javax.persistence.*;
import java.util.Date;
/**
* 小区周边配套统计信息
*
* @author shenjian@fangjia.com
*/
@Entity
@Table(name = "fangjia_stat.community_poi_gj")
public class CommunityPoi implements MongoEntity {
private static final long serialVersionUID = -7663644539542318409L;
private String id;
private String city;
private String region;
private String name;
private Integer subway;
private Integer traffic;
private Integer bank;
private Integer primary;
private Integer junior;
private Integer hospital;
/**
* 不利设施,丧葬
*/
private Integer funeral;
@Id
@Column(name = "_id")
@Override
public String getId() {
return id;
}
@Override
public void setId(String id) {
this.id = id;
}
@Column(name = "city")
public String getCity() {
return city;
}
public void setCity(String city) {
this.city = city;
}
@Column(name = "region")
public String getRegion() {
return region;
}
public void setRegion(String region) {
this.region = region;
}
@Column(name = "name")
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
@Column(name = "subway")
public Integer getSubway() {
return subway;
}
public void setSubway(Integer subway) {
this.subway = subway;
}
@Column(name = "traffic")
public Integer getTraffic() {
return traffic;
}
public void setTraffic(Integer traffic) {
this.traffic = traffic;
}
@Column(name = "bank")
public Integer getBank() {
return bank;
}
public void setBank(Integer bank) {
this.bank = bank;
}
@Column(name = "primary")
public Integer getPrimary() {
return primary;
}
public void setPrimary(Integer primary) {
this.primary = primary;
}
@Column(name = "junior")
public Integer getJunior() {
return junior;
}
public void setJunior(Integer junior) {
this.junior = junior;
}
@Column(name = "hospital")
public Integer getHospital() {
return hospital;
}
public void setHospital(Integer hospital) {
this.hospital = hospital;
}
@Column(name = "funeral")
public Integer getFuneral() {
return funeral;
}
public void setFuneral(Integer funeral) {
this.funeral = funeral;
}
}
@@ -67,12 +67,7 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
must.add(addTerms("district_name", alias));
must.add(addMatch("cat", "sell"));
must.add(addRange("avg_price", minPrice, maxPrice));
JSONObject term = new JSONObject();
JSONObject match = new JSONObject();
match.put("p_date", 20210501);
term.put("match", match);
must.add(term);
must.add(addRangeGte("p_date", 20210501));
bool.put("must", must);
query.put("bool", bool);
@@ -168,6 +163,16 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
return term;
}
private JSONObject addRangeGte(String name, Number min){
JSONObject term = new JSONObject();
JSONObject range = new JSONObject();
JSONObject rangeValue = new JSONObject();
rangeValue.put("gte", min);
range.put(name, rangeValue);
term.put("range", range);
return term;
}
private List<Seawater> wrapSeawater(List<JSONObject> jsonList){
List<Seawater> seawaterList= new ArrayList<Seawater>();
@@ -210,6 +215,11 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
seawater.setFloor(source.getInt("floor"));
}catch(Exception e){
seawater.setFloor(0);
}
try{
seawater.setFloorInfo(source.getString("floor_info"));
}catch(Exception e){
}
if(source.containsKey("fitment")){
seawater.setFitment(source.getString("fitment"));
@@ -48,6 +48,7 @@ public class Seawater implements SeawaterEntity {
private Double area;
private Integer height;
private Integer floor;
private String floorInfo;
private String fitment;
private Integer room;
private Integer hall;
@@ -198,6 +199,15 @@ public class Seawater implements SeawaterEntity {
this.floor = floor;
}
@Column(name = "floorInfo")
public String getFloorInfo() {
return floorInfo;
}
public void setFloorInfo(String floorInfo) {
this.floorInfo = floorInfo;
}
@Column(name = "fitment")
public String getFitment() {
return fitment;
@@ -31,12 +31,15 @@ import javax.validation.Valid;
import com.fangjia.core.util.*;
import com.fangjia.dal.*;
import com.fangjia.dal.mongo.model.*;
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
import com.fangjia.dal.mongo.open.model.CommunityPoi;
import com.fangjia.dal.seawater.dao.SeawaterDao;
import com.fangjia.dal.seawater.model.Seawater;
import com.fangjia.open.dto.*;
import com.fangjia.service.district.*;
import com.fangjia.service.mongo.fangjia.*;
import com.fangjia.service.open.*;
import com.fangjia.service.open.impl.CommunityPoiServiceImpl;
import com.fangjia.service.util.DistrictScoreUtil;
import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
@@ -175,6 +178,9 @@ public class PropertyController {
private SeawaterDao seawaterDao;
private SeawaterService seawaterService;
private SqmetreRentService sqmetreRentService;
private CommunityPoiService communityPoiService;
private CommunityBasePriceService communityBasePriceService;
private EvaluateFactorService evaluateFactorService;
private FileDao fileDao = FileDao.getInstance();
private static final int DEFAULT_PAGE_SIZE = 20;
@@ -390,10 +396,31 @@ public class PropertyController {
put("5000", "five");// 五线
}
};
/**
* 第三方均价映射
*/
private static final Map<String, String> OTHER_PRICE_MAP = new HashMap<String, String>() {
{
put("zhuge_price", "诸葛找房");
put("anjuke_price", "安居客");
put("fang_price", "房天下");
put("lianjia_price", "链家");
put("cityhouse_price", "禧泰数据");
put("maitian_price", "麦田");
put("ke_price", "贝壳");
put("taofang_price", "淘房365");
put("zhongyuan_price", "中原");
put("tbs_price", "兔博士");
put("qfang_price", "Q房");
put("fangtan_price", "房探");
put("fangduoduo_price", "房多多");
put("wiwj_price", "我爱我家");
}
};
/**
* 利用百度API把GPS转换成Google经纬度
*
*
* @param lng
* @param lat
* @return
@@ -415,7 +442,7 @@ public class PropertyController {
/**
* 利用算法模板把GPS转换成Google经纬度
*
*
* @param lng
* @param lat
* @return
@@ -607,7 +634,7 @@ public class PropertyController {
/**
* 判断小区是否有经纬度
*
*
* @param district
* @return
*/
@@ -618,7 +645,7 @@ public class PropertyController {
/**
* 调用匹配接口匹配小区
*
*
* @param city
* @param address
* @param region
@@ -992,6 +1019,32 @@ public class PropertyController {
}
}
// 调整面积不同面积价格
try {
DistrictStatBo districtStatBo = districtStatService.findDistrictStat(new DistrictKey(district));
int orgAvgPrice = 0;
if (districtStatBo != null) {
if (StringUtils.isNotBlank(district.getEstateType2()) && district.getEstateType2().contains("别墅") && districtStatBo.getVillaDistrictStat() != null) {
orgAvgPrice = districtStatBo.getVillaDistrictStat().getAvgPrice();
} else if (districtStatBo.getDistrictStat() != null) {
orgAvgPrice = districtStatBo.getDistrictStat().getAvgPrice();
}
}
if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) {
int factorPrice = evaluation.getAvgPrice();
int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice);
Integer minPrice = (int)(evaluation.getAvgPrice() * 0.95);
Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.05);
// 偏离度超过5%不进行调整
if (minPrice <= avgPrice && avgPrice <= maxPrice) {
evaluation.setAvgPrice(avgPrice);
evaluation.setBigTotalPrice((long) (avgPrice * evaluation.getArea()));
}
}
} catch (Exception ex) {
}
if (evaluation.getTotalPrice() != null && evaluation.getTotalPrice() != 0
&& (cityPriceCredit == null || evaluation.getCredit() >= cityPriceCredit)) {
Map<String, Object> districtPriceMap = new HashMap<String, Object>();
@@ -1162,19 +1215,52 @@ public class PropertyController {
evaluationService.remove(evaluation.getId());
mav.setModel(BasicModel.buildOKModel().setMsg("暂时不支持该小区的估价").markNoData());
return mav;
}
// 调整面积不同面积价格
try {
DistrictStatBo districtStatBo = districtStatService.findDistrictStat(new DistrictKey(district));
int orgAvgPrice = 0;
if (districtStatBo != null) {
if (StringUtils.isNotBlank(district.getEstateType2()) && district.getEstateType2().contains("别墅") && districtStatBo.getVillaDistrictStat() != null) {
orgAvgPrice = districtStatBo.getVillaDistrictStat().getAvgPrice();
} else if (districtStatBo.getDistrictStat() != null) {
orgAvgPrice = districtStatBo.getDistrictStat().getAvgPrice();
}
}
if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) {
int factorPrice = evaluation.getAvgPrice();
int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice);
Integer minPrice = (int)(evaluation.getAvgPrice() * 0.95);
Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.05);
// 偏离度超过5%不进行调整
if (minPrice <= avgPrice && avgPrice <= maxPrice) {
evaluation.setAvgPrice(avgPrice);
evaluation.setBigTotalPrice((long) (avgPrice * evaluation.getArea()));
}
}
} catch (Exception ex) {
}
Map<String, Object> districtPriceMap = new LinkedHashMap<String, Object>();
districtPriceMap.put("city", district.getCity());
districtPriceMap.put("district", district.getRegion());
districtPriceMap.put("name", district.getName());
long totalPrice = evaluation.getBigTotalPrice();
if(CITY_CORRECTION_FACTOR.containsKey(district.getCity())){
totalPrice = (long)(totalPrice * (1 - CITY_CORRECTION_FACTOR.get(district.getCity())));
evaluation.setBigTotalPrice(totalPrice);
evaluation.setAvgPrice((int)(totalPrice/evaluation.getArea()));
}
// 取消系数下调
// if(CITY_CORRECTION_FACTOR.containsKey(district.getCity())){
// totalPrice = (long)(totalPrice * (1 - CITY_CORRECTION_FACTOR.get(district.getCity())));
// evaluation.setBigTotalPrice(totalPrice);
// evaluation.setAvgPrice((int)(totalPrice/evaluation.getArea()));
// }
districtPriceMap.put("totalPrice", evaluation.getBigTotalPrice());
districtPriceMap.put("unitPrice", evaluation.getAvgPrice());
// 新增置信度
CommunityBasePrice communityBasePrice = communityBasePriceService.getById(district.getId());
if (communityBasePrice != null) {
districtPriceMap.put("credit", communityBasePrice.getCredit());
} else {
districtPriceMap.put("credit", evaluation.getCredit());
}
List<String> names = new ArrayList<String>();
names.add(district.getName());
if(district.getAlias() != null){
@@ -1186,6 +1272,31 @@ public class PropertyController {
}
Integer minPrice = (int)(evaluation.getAvgPrice() * 0.8);
Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.2);
Integer min10Price = (int)(evaluation.getAvgPrice() * 0.9);
Integer max10Price = (int)(evaluation.getAvgPrice() * 1.1);
// 新增第三方均价
try {
double dCredit = 0.61;
Map<String, Object> rMap = getLastEvaluationPrice(district, evaluation, dCredit);
if (null != rMap) {
if (rMap.containsKey("other_price")) {
Map<String, Integer> otherPriceMap = (Map<String, Integer>) rMap.get("other_price");
Map<String, Integer> priceResultMap = new HashMap<String, Integer>();
for (String key : otherPriceMap.keySet()) {
Integer price = otherPriceMap.get(key);
// 剔除偏差大于10的
if (min10Price <= price && max10Price >= price) {
if (OTHER_PRICE_MAP.containsKey(key)) {
priceResultMap.put(OTHER_PRICE_MAP.get(key), price);
}
}
}
districtPriceMap.put("otherPrice", priceResultMap);
}
}
} catch (Exception ex) {
}
List<Seawater> houses = seawaterDao.findHouses(district.getCity(), district.getRegion(), names,minPrice, maxPrice, 20);
if(houses.size() > 3){
districtPriceMap.put("calcType", 1);
@@ -1214,7 +1325,7 @@ public class PropertyController {
}else if(district.getLng() != null && district.getLng() > 0){
double lng = DataUtil.roundDouble(district.getLng() / 10000000000.0,6);
double lat = DataUtil.roundDouble(district.getLat() / 10000000000.0, 6);
List<District> districts = districtService.findNearestDistrict(district.getEstateType2(), lng, lat, 1000 * 3 , 4, null);
List<District> districts = districtService.findNearestDistrict(district.getEstateType2(), lng, lat, 1000 * 3 , 6, null);
if(districts.size() < 2){
evaluationService.remove(evaluation.getId());
mav.setModel(BasicModel.buildOKModel().setMsg("暂时不支持该小区的估价").markNoData());
@@ -1252,9 +1363,15 @@ public class PropertyController {
districtDto.append("buildYear", String.valueOf(buildYear));
}
districtAnalyData = districtAnalyDataService.findByDistrictKey(new DistrictKey(d));
if(districtAnalyData != null && districtAnalyData.getHalfYearMarkup() != null){
districtDto.append("halfYearMarkup",DataUtil.roundDouble( districtAnalyData.getHalfYearMarkup() * 100 ,2));
if(districtAnalyData != null){
if (districtAnalyData.getHalfYearMarkup() != null) {
districtDto.append("halfYearMarkup", DataUtil.roundDouble(districtAnalyData.getHalfYearMarkup() * 100, 2));
}
if (districtAnalyData.getAvgPrice() != null && minPrice <= districtAnalyData.getAvgPrice() && maxPrice >= districtAnalyData.getAvgPrice()) {
districtDto.append("avgPrice", districtAnalyData.getAvgPrice());
}
}
nearCommunityDtos.add(districtDto);
}
Integer buildYear = districtService.getBuildYear(district);
@@ -1262,6 +1379,18 @@ public class PropertyController {
modelFeaturesFto.put("buildYear", String.valueOf(buildYear));
}
modelFeaturesFto.append("nearCommunity", nearCommunityDtos);
// 模型特征新增周边数量
CommunityPoi communityPoi = communityPoiService.getById(district.getId());
if (communityPoi != null) {
modelFeaturesFto.put("subway", communityPoi.getSubway());
modelFeaturesFto.put("traffic", communityPoi.getTraffic());
modelFeaturesFto.put("bank", communityPoi.getBank());
modelFeaturesFto.put("primary", communityPoi.getPrimary());
modelFeaturesFto.put("hospital", communityPoi.getHospital());
modelFeaturesFto.put("junior", communityPoi.getJunior());
modelFeaturesFto.put("funeral", communityPoi.getFuneral());
}
districtPriceMap.put("modelFeatures", modelFeaturesFto);
}
mav.setModel(BasicModel.buildOKModel().setResult(districtPriceMap));
@@ -1469,7 +1598,7 @@ public class PropertyController {
return mav;
}
}
/**
* 四川锦程 定制接口 多建筑年代
* @param model
@@ -1581,7 +1710,7 @@ public class PropertyController {
districtPriceMap.put("buildYear", matcher3.group());
}
}
// 根据城市等 级设置D级可信度
ApiUserAuthority apiUserAuthority = apiUserAuthorityService
@@ -2094,7 +2223,7 @@ public class PropertyController {
/**
* 小区租金查询接口
*
*
* @param model
* @return
*/
@@ -2573,7 +2702,7 @@ public class PropertyController {
/**
* 批量获取小区的均价,均价走势数据
*
*
* @param request
* @return
*/
@@ -2715,7 +2844,7 @@ public class PropertyController {
/**
* 批量获取小区的配套数据,最多12个小区
*
*
* @param request
* @return
*/
@@ -2860,7 +2989,7 @@ public class PropertyController {
/**
* 获取JSON参数中的num参数值
*
*
* @param jo
* @return
*/
@@ -2885,7 +3014,7 @@ public class PropertyController {
/**
* 获取JSON参数中的radius值
*
*
* @param jo
* @return
*/
@@ -3693,7 +3822,7 @@ public class PropertyController {
/**
* 解析出关键字中的小区名
*
*
* @param keyword
* @return
*/
@@ -3810,7 +3939,7 @@ public class PropertyController {
/**
* 小区统计分析接口
*
*
* @param model
* @return
*/
@@ -3895,7 +4024,7 @@ public class PropertyController {
/**
* 小区交通接口
*
*
* @param model
* @return
*/
@@ -3953,7 +4082,7 @@ public class PropertyController {
/**
* 小区周边接口
*
*
* @param model
* @return
*/
@@ -4012,7 +4141,7 @@ public class PropertyController {
/**
* 相同小区相似户型房源挂牌量接口<br>
* 获取该小区的满足指定相似条件的挂牌房源量 条件说明: 1.输入面积-5<=房源面积<=输入面积+5 2.上个月发布的房源
*
*
* @author yinjihuan
* @date 2016年9月1日
* @param city
@@ -4052,7 +4181,7 @@ public class PropertyController {
/**
* 周边小区相似户型挂牌量接口<br>
* 获取该小区的满足指定相似条件的挂牌房源量 条件说明: 1.输入面积-5<=房源面积<=输入面积+5 2.上个月发布的房源
*
*
* @author yinjihuan
* @date 2016年9月1日
* @param city
@@ -4103,7 +4232,7 @@ public class PropertyController {
/**
* 小区周边一公里均价
*
*
* @author wenyan
* @param model
* @return
@@ -4126,7 +4255,7 @@ public class PropertyController {
/**
* 相同小区相似户型租金范围查询接口
*
*
* @param model
* @return
*/
@@ -4169,7 +4298,7 @@ public class PropertyController {
/**
* 周边小区相似户型租金范围查询接口
*
*
* @param model
* @return
*/
@@ -4232,7 +4361,7 @@ public class PropertyController {
/**
* 室号查询接口
*
*
* @param model
* @return
*/
@@ -4268,7 +4397,7 @@ public class PropertyController {
/**
* 楼栋楼层信息查询接口
*
*
* @param model
* @return
*/
@@ -4293,7 +4422,7 @@ public class PropertyController {
/**
* 楼栋查询接口
*
*
* @param model
* @return
*/
@@ -4338,7 +4467,7 @@ public class PropertyController {
/**
* 楼栋查询接口
*
*
* @param model
* @return
*/
@@ -4791,7 +4920,7 @@ public class PropertyController {
/**
* 房产交易记录查询
*
*
* @param model
* @return
*/
@@ -4851,7 +4980,7 @@ public class PropertyController {
/**
* 房产拍卖记录
*
*
* @param model
* @return
*/
@@ -4912,7 +5041,7 @@ public class PropertyController {
/**
* 房产近期评估次数
*
*
* @param model
* @return
*/
@@ -5627,4 +5756,16 @@ public class PropertyController {
public void setRentDealPriceService(RentDealPriceService rentDealPriceService) {
this.rentDealPriceService = rentDealPriceService;
}
public void setCommunityPoiService(CommunityPoiService communityPoiService) {
this.communityPoiService = communityPoiService;
}
public void setEvaluateFactorService(EvaluateFactorService evaluateFactorService) {
this.evaluateFactorService = evaluateFactorService;
}
public void setCommunityBasePriceService(CommunityBasePriceService communityBasePriceService) {
this.communityBasePriceService = communityBasePriceService;
}
}
@@ -75,8 +75,8 @@ public class AnalyticsFilter implements Filter {
if ("1".equals(request.getParameter("__refresh_cache"))) {
threadLocalData.put(ThreadLocalKey.DisableCache, "1");
}
createUID(request, response);
createSID(request, response);
//createUID(request, response);
//createSID(request, response);
response.addHeader("Access-Control-Allow-Origin", "*");
response.addHeader("Access-Control-Allow-Methods", "*");
chain.doFilter(request, response);
@@ -27,7 +27,7 @@ import java.util.regex.Pattern
@BatchOwner(Watcher.GaoJuan)
class BigdataCommunityBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
static main(args) {
runBatch(new BigdataCommunityBatch(), args);
@@ -187,11 +187,22 @@ class BigdataCommunityBatch extends BaseBatch{
}else{
def jsonObj = null
if(dist.address){
jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name)
// jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name)
jsonObj = OpenApiClient.standard1Action(dist.city, dist.address)
}
if(jsonObj && jsonObj.searchAddress){
ret << jsonObj.searchAddress.street
ret << jsonObj.searchAddress.streetNumber
if(jsonObj && jsonObj.street){
if(jsonObj.street){
ret << jsonObj.street
}else{
ret << ""
}
if(jsonObj.streetNumber){
ret << jsonObj.streetNumber
}else{
ret << ""
}
}else{
ret << ""
ret << ""
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220318, 20220419);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220418, 20220519);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
//测试通过匹配接口配房源效果
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
@@ -97,7 +97,7 @@ class BigdataForsaleBatch extends BaseBatch{
totalCount.addAndGet(1)
def commId = rMap.get("_id").toString()
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias)
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias, rMap.estate_type2)
//def ret = findHousesByDistId(city, commId, "sell", startDate,endDate)
// if(ret && ret.size()<100){//导历史房源需要,以后不需要
// ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias)
@@ -408,7 +408,7 @@ class BigdataForsaleBatch extends BaseBatch{
}
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
@@ -449,6 +449,13 @@ class BigdataForsaleBatch extends BaseBatch{
def map = houses[k];
DBObject seawater = new BasicDBObject(map)
if(estate_type2 && estate_type2=="ÆÕͨסլ"){
if(seawater.estate_type&& seawater.estate_type.contains("ÉÌס")){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
}
if(!seawater.total_price){
continue;
}
@@ -57,6 +57,12 @@ class CcbFtpUtils {
ftpUpload(dataCodeFileName)//上传加密码文件到FTP
}
static def listFileNames(month){
//上传到ftp
sftpChannel = reConnect();
Vector files = sftp.listFiles(ftpPath+ month+"/", sftpChannel);
return files
}
static main(args) {
@@ -15,8 +15,8 @@ class DealPriceDataProcess {
def sqlInstance2 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
// def sqlInstance2 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
@@ -373,16 +373,17 @@ class DealPriceDataProcess {
def citys = CcbCommonUtils.citys
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def publishTime = '2021-10-01'
def month = 202203
def month = 202204
def rows = []
for(def city : citys) {
def rets = findPeGteThirdParty(city, 202204)
def rets = findPeGteThirdParty(city, 202205)
def total = 0
println city +" "+rets.size()
for(def m : rets){
def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
sqlInstance236.executeUpdate(sql)
}
println city +" "+rets.size()
}
}
@@ -38,8 +38,8 @@ class ExportOracleData {
exportCommunityBuileType()
exportThirdPartyPrice2(lastMonth)
exportBigdataForsale("2022-03-19")
exportBigdataDeal("2022-03-01")
exportBigdataForsale("2022-04-19")
exportBigdataDeal("2022-04-01")
//计算结果相关
@@ -3,8 +3,10 @@ package com.fangjia.script.ccbevaluation
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.jcraft.jsch.ChannelSftp
import groovy.sql.Sql
import com.fangjia.script.ccbevaluation.ronghe.SSHHelper
import org.apache.commons.codec.digest.DigestUtils
class ExportOracleEvaluationData {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
@@ -33,6 +35,8 @@ class ExportOracleEvaluationData {
exportCommCaseDetailGj(curMonth)
def curDate = curMonth.toString()+"28"
// 生成MD5文件
exportMd5()
SSHHelper jschCommand = new SSHHelper();
System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
// okÎļþ
@@ -413,4 +417,23 @@ class ExportOracleEvaluationData {
CcbFtpUtils.ftpUpload(null, dataCodeFileName, curDate)
}
def exportMd5(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
Vector files = CcbFtpUtils.listFileNames(curDate)
Iterator fileVec = files.iterator()
def fileRows = []
while(fileVec.hasNext()){
ChannelSftp.LsEntry file = fileVec.next()
if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){
fileRows << file.getFilename() +":"+ DigestUtils.md5Hex(file.getFilename())
}
}
def fileName = "/ccb/md5.txt"
if(CommonDatautils.isWindow()){
fileName = "D:\\房价网\\md5.txt"
}
FileUtils.createTxtFile(fileRows, fileName, "utf-8", false);
CcbFtpUtils.ftpUpload(null, fileName, curDate)
println "md5.txt 上传Ftp完成......"
}
}
@@ -1,21 +1,29 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.fangjia.core.RegionData
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import groovy.sql.Sql
import org.bson.types.ObjectId
class LoadGuide {
public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new LoadGuide()
task.run()
}
def run(){
addPersonGuided("D:\\数据\\建行\\人工指导价更新.xlsx")
// match("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
// saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞指导价218.xlsx")
@@ -26,9 +34,51 @@ class LoadGuide {
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\无锡100 广州96 二手住房成交参考价格表.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx")
savePrice("D:\\数据\\建行\\房产估值\\指导价\\三亚指导价30.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\三亚指导价30.xlsx")
}
def addPersonGuided(filename){
ExcelUtils.modify(filename, { d ->
def city = d['B']
if(city){
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
createSql( d, cityCodeDatas, distCodeMap)
}
})
}
def createSql(d, cityCodeDatas, distCodeMap){
def id = new ObjectId()
def commid = d['A']
def cityId = cityCodeDatas.get(d['B'])
def sampleRegion = d['C'].replace("街道","").replace("镇","")
def distRegion = distCodeMap.get(sampleRegion)
def cityName = d['B']
def distName = d['C']
def commName = d['D']
def price = d['E'].toDouble().toInteger()
def dist = CommonDatautils.findDistrict(cityName, sampleRegion, commName, "district")
if(dist){
def createtime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
def dealDate = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd")
def sql = "insert into guided_price_gj(ID,COMMID,COMMNAME,CITYID,CITYNAME,DISTID,DISTNAME,DEALDATE,AVG_PRICE,GUIDING_PERSON,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES " +
"('${id.toString()}','${dist._id.toString()}', '${commName}','${cityId}','${cityName}','${distRegion.district_code}','${distName}',to_date('${dealDate}','yyyy-mm-dd'),${price},'房价网','',to_date('${createtime}','yyyy-mm-dd hh24:mi:ss'),to_date('${createtime}','yyyy-mm-dd hh24:mi:ss'),'FJW')"
println sql
sqlInstance.executeUpdate(sql)
}else{
println " 没查到 "+commName
}
}
def match(def filename){
def city = "西安"
ExcelUtils.modify(filename,{d->
@@ -0,0 +1,67 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.core.util.DistrictCodeUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import groovy.sql.Sql
import org.apache.lucene.queryparser.classic.QueryParser
import org.apache.poi.ss.util.RegionUtil
import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* oracle中小区统计数据同步至85中
*/
class OracleDistrict2Mongo85 {
static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static final DBCollection communityPoiConn = MongoUtils.getCollectionByName("fangjia_stat", "community_poi_gj", "192.168.0.85", 28017)
static main(args) {
// 只写入重庆市数据
def sql = "select cityid, distname, commid, commname, subway, traffic, bank, primary, junior, hospital, funeral from COMMUNITY_POI_GJ"
def count = 0
sqlInstance.eachRow(sql, { r ->
def city = DistrictCodeUtils.getCityName(r['cityid'])
def region = RegionData.formatRegion(city, r['distname'])
DBObject dbObject = new BasicDBObject("_id", r['commid'])
DBObject updateObject = new BasicDBObject("city", city).append("region", region)
.append("name", r['commname'])
.append('subway', Integer.parseInt(r['subway'].toString()))
.append("traffic", Integer.parseInt(r['traffic'].toString()))
.append("bank", Integer.parseInt(r['bank'].toString()))
.append("primary", Integer.parseInt(r['primary'].toString()))
.append("junior", Integer.parseInt(r['junior'].toString()))
.append("hospital", Integer.parseInt(r['hospital'].toString()))
.append("funeral", Integer.parseInt(r['funeral'].toString()))
.append("c_date", new Date());
communityPoiConn.update(dbObject, updateObject, true, false)
count++;
if (count % 500 == 0) {
println "已写入" + count + "条小区数据"
}
});
}
}
@@ -0,0 +1,52 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.core.util.DistrictCodeUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import groovy.sql.Sql
/**
* oracle中小区置信度数据同步至85中
*/
class OracleDistrictBasePrice2Mongo85 {
static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static final DBCollection communityBasePriceConn = MongoUtils.getCollectionByName("business", "community_base_price_gj", "192.168.0.85", 28017)
static main(args) {
def sql = "SELECT cityname, distname, commid, commname, confidence FROM community_base_price_gj WHERE YEAR_MONTH='202205'"
def count = 0
sqlInstance.eachRow(sql, { r ->
def city = r['cityname']
def region = RegionData.formatRegion(city, r['distname'])
DBObject dbObject = new BasicDBObject("_id", r['commid'])
if (r['confidence'] != null) {
def credit = new BigDecimal(r['confidence']).setScale(2, BigDecimal.ROUND_HALF_UP).toDouble()
DBObject updateObject = new BasicDBObject("city", city).append("region", region)
.append("name", r['commname'])
.append("credit", credit)
.append("c_date", new Date());
communityBasePriceConn.update(dbObject, updateObject, true, false)
count++;
} else {
print "置信度为空:" + r.toString()
}
if (count % 500 == 0) {
println "已写入" + count + "条小区数据"
}
});
}
}
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
@BatchOwner(Watcher.GaoJuan)
class ThirdPartyPriceBatch extends BaseBatch {
def sDate= 202203//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202201
def endDate = 202203
def sDate= 202204//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202202
def endDate = 202204
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
@@ -40,8 +40,8 @@ class TradeStatLog {
static main(args) {
def task = new TradeStatLog()
// task.run()//统计最近3个月抓取的成交数据量
task.exportTrade()//导出未入库成交
task.run()//统计最近3个月抓取的成交数据量
// task.exportTrade()//导出未入库成交
// task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
// task.averageStat()
@@ -75,13 +75,13 @@ class TradeStatLog {
def run(){
def citys = CcbCommonUtils.citys;
def rets = []
rets << ["城市","1月总量","2月总量","3月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
rets << ["城市","2月总量","3月总量","4月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2022-01-01"
def date = "2022-02-01"
def endDate = "2022-03-01"
def newDate = "2022-04-01"
def startDate = "2022-02-01"
def date = "2022-03-01"
def endDate = "2022-04-01"
def newDate = "2022-05-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -131,7 +131,7 @@ class TradeStatLog {
}
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-03成交统计.xlsx", rets, "数据")
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-04成交统计.xlsx", rets, "数据")
println "表格生成完成"
}
@@ -0,0 +1,155 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
class CcbDataStat {
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new CcbDataStat()
// task.run()
task.tradeRun()
}
def tradeRun(){
def ccbCitys = CcbCommonUtils.citys;
def startDate = DateUtils.parseFromTightByPattern("2021-01-01","yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd")
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\成交案例统计_0531.xlsx", { d ->
if(d['A']){
if(ccbCitys.contains(d['A'])){
d['D'] = findOracleTotalCount(d['A'])
}else{
def totalcount = findAllTradeCount(d['A'], startDate, endDate)
def repeatCount = findRepeatCount(d['A'], startDate, endDate)
def matchTotalCount = findMatchTradeCount(d['A'], startDate, endDate)
def matchRepeatTotalCount = findMatchRepeatTradeCount(d['A'], startDate, endDate)
d['B'] = totalcount
d['C'] = repeatCount
d['D'] = matchTotalCount
d['E'] = matchRepeatTotalCount
}
}
})
}
def findRepeatCount(city, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
def dbcursor = DEAL_PRICE_COLL.find(q)
def set = new HashSet()
for(def d : dbcursor){
def key = d.region+"##"+d.district_name+"##"+d.avg_price+"##"+d.area+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyyMMdd")
if(!set.contains(key)){
set << key
}
}
return set.size()
}
def findMatchRepeatTradeCount(city, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
q.put("match_id", new BasicDBObject('$exists', true))
def dbcursor = DEAL_PRICE_COLL.find(q)
def set = new HashSet()
for(def d : dbcursor){
def key = d.match_id+"#"+d.avg_price+"##"+d.area+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyyMMdd")
if(!set.contains(key)){
set << key
}
}
return set.size()
}
def findMatchTradeCount(city, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
q.put("match_id", new BasicDBObject('$exists', true))
def count = DEAL_PRICE_COLL.count(q)
return count
}
public static def findAllTradeCount(city, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
// q.put("match_id", new BasicDBObject('$exists', false))
def total = DEAL_PRICE_COLL.count(q)
return total
}
def findOracleTotalCount(city){
def sql = "select count(*) as count from bigdata_deal_gj where cityname='${city}' and dealdate>=to_date('2021-01-01','yyyy-MM-dd') and dealdate<to_date('2022-01-01','yyyy-MM-dd')";//where cityId in (350200,150100)
def count = 0
sqlInstance.eachRow(sql, { r ->
count = r.count
})
return count
}
def run(){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\建行成交\\92个城市字段覆盖率情况.xlsx",{ d->
if(d['A']) {
def citycode = cityCodeDataMap.get(d['A'])
def total = findTotalCount(citycode)
def f_count = findDistFinishDate(citycode)
def b_count = findBuildTypeCount(citycode)
def f_pre = DataHandleUtils.getDoubleRound(2, f_count*100/total)
def b_pre = DataHandleUtils.getDoubleRound(2, b_count*100/total)
d['C'] = f_pre
d['D'] = b_pre
d['E'] = total
println d['A'] +" "+ total +" "+f_count+" "+b_count
}
})
}
def findTotalCount(citycode){
def sql = "select count(*) as count from bigdata_community_gj where cityid='${citycode}' and status=0";//where cityId in (350200,150100)
def count = 0
sqlInstance.eachRow(sql, { r ->
count = r.count
})
return count
}
def findDistFinishDate(citycode){
def sql = "select count(*) as count from bigdata_community_gj where cityid='${citycode}' and status=0 and BUILDYEAR>0";//where cityId in (350200,150100)
def count = 0
sqlInstance.eachRow(sql, { r ->
count = r.count
})
return count
}
def findBuildTypeCount(citycode){
def sql = "select count(*) as count from (select commid, count(commid) as count from bigdata_commbuildtype_gj where cityid='${citycode}' and BUILDTYPE!='0199' group by commid)";//where cityId in (350200,150100)
def count = 0
sqlInstance.eachRow(sql, { r ->
count = r.count
})
return count
}
}
@@ -0,0 +1,117 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import groovy.sql.Sql
class CcbDistNameMatch {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new CcbDistNameMatch();
// task.run()
task.run1()
}
def run1(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\新增映射别名.xlsx", { d ->
if (d['A']) {
def region = d['B']
def newRegion = RegionData.formatRegion(d['A'], region)
if (newRegion) {
if(d['D'].contains("、")){
println d['D']
}
def arrs = d['D'].split("、")
def list = []
def flags = []
for(def s : arrs){
def dist = CommonDatautils.findDistrict(d['A'], newRegion, s, "district")
if(dist){
list << dist.name
flags << "完全一致"
}else{
def jsonObject = OpenApiClient.addressMatch(d['A'], s, "property", false, newRegion)
if(jsonObject != null && jsonObject.precise!=-1) {
if(jsonObject.get("district") == newRegion){
list << jsonObject.get("name")
flags << "匹配接口"
}
}
}
}
if(list){
list = list.unique()
d['E'] = list.join(",")
d['F'] = flags.join(",")
}
}
}
})
}
def run(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\不同小区.xlsx", { d ->
def regionCode = null
if (d['A']) {
def region = d['B']
def newRegion = RegionData.formatRegion(d['A'], region)
if (newRegion) {
def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
if(dist){
d['D'] = dist.name
def count = findDistrictByOracle(dist._id.toString())
if(count && count>0){
d['F'] = 1
}else{
d['H'] = dist.estate_type2
d['J'] = dist.name_cat
}
}else{
def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['C'], "district")
if (rets) {
if(rets.size()>1){
d['E'] = "别名-多个"
}else{
dist = rets[0]
d['E'] = d['C'] +"--"+dist.name
def count = findAliasByOracle(dist._id.toString())
if(count && count>0) {
d['G'] = 1
}else{
d['H'] = dist.estate_type2
d['J'] = dist.name_cat
}
}
}
}
}
}
});
}
def findDistrictByOracle(commid){
def sql = "select count(*) as count from bigdata_community_gj where commid='${commid}' and status=0";
def count = 0
sqlInstance.eachRow(sql,{r->
count = r.count
});
return count
}
def findAliasByOracle(commid){
def sql = "select count(*) as count from address_alias_fjw where address_id='${commid}'";
def count = 0
sqlInstance.eachRow(sql,{r->
count = r.count
});
return count
}
}
@@ -7,6 +7,7 @@ import com.fangjia.core.util.GeoUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.gj.district.NameProcess
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.CommonUtils
@@ -26,9 +27,9 @@ class CcbDistrictToDB {
// task.run()
// task.run1()
// task.meituanRun()
// task.meituanSqrentRun()
task.meituanSqrentRun()
// task.updateDistrict()
task.updateDongguangRegion()
// task.updateDongguangRegion()
}
def updateDongguangRegion(){
@@ -80,13 +81,17 @@ class CcbDistrictToDB {
}
def meituanSqrentRun(){
ExcelUtils.modify("D:\\数据\\美团\\美团需入库小区合并后.xlsx", { d ->
def index = 0
ExcelUtils.modify("D:\\数据\\美团\\美团待入库.xlsx", { d ->
index++
println index+" "+d['H']
def city = d['F']
def region = d['G']
def newRegion = RegionData.formatRegion(city, region)
if(newRegion) {
def name = d['H']
def name = NameProcess.nameStandard(city, newRegion, d['H'])
name = nameFileter(name)
if(name){
def estateType2 = "普通住宅"
if(name.contains("别墅")){
@@ -103,7 +108,6 @@ class CcbDistrictToDB {
}
}
}
def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1));
CommonUtils.recalculationSqRent(city, region, name, estateType2, lastMonthDay,false,0)
}
@@ -115,97 +119,100 @@ class CcbDistrictToDB {
def index = 0//美团需入库小区合并后.xlsx
def keys = new CopyOnWriteArrayList()
ExcelUtils.multiThreadingModify("D:\\数据\\美团\\美团需入库小区合并后.xlsx", { d ->
ExcelUtils.multiThreadingModify("D:\\数据\\美团\\美团待入库.xlsx", { d ->
index++
println index+" "+d['H']
def city = d['F']
def region = d['G']
def newRegion = RegionData.formatRegion(city, region)
if(newRegion){
def name = d['H']
if(newRegion) {
def name = NameProcess.nameStandard(city, newRegion, d['H'])
name = nameFileter(name)
if (name) {
def formatAddress = null
if(d['E']){
formatAddress = AddressUtil.formatAddress(city, d['E'])
if (d['I']) {
formatAddress = AddressUtil.formatAddress(city, d['I'])
}
if(!formatAddress || (formatAddress && !formatAddress.contains("号") && !formatAddress.contains("交叉"))){
def rets = OpenApiUtils.getGaoDeSearchLocation(city, region, name)
def map = findStandardObject2(city,region, name, d['E'], rets);
if(map && map.address){
formatAddress = AddressUtil.formatAddress(city, map.address)
if(!formatAddress){
if(map.address.contains("交叉口")){
formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉口")+3)
}else if(map.address.contains("交叉")){
formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉")+2)
}
}
}
}
// if(!formatAddress || (formatAddress && !formatAddress.contains("号") && !formatAddress.contains("交叉"))){
// def rets = OpenApiUtils.getGaoDeSearchLocation(city, region, name)
// def map = findStandardObject2(city,region, name, d['E'], rets);
// if(map && map.address){
// formatAddress = AddressUtil.formatAddress(city, map.address)
// if(!formatAddress){
// if(map.address.contains("交叉口")){
// formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉口")+3)
// }else if(map.address.contains("交叉")){
// formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉")+2)
// }
// }
// }
// }
Long glat = null
Long glng = null
Long blat = null
Long blng = null
if(d['K']){
glat = d['L'].toString().toDouble() * 10000000000L
glng = d['K'].toString().toDouble() * 10000000000L
def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3)
if (d['L']) {
glat = d['M'].toString().toDouble() * 10000000000L
glng = d['L'].toString().toDouble() * 10000000000L
def baidu = BaiDuApiUtils.geoconv(d['L'].toString().toDouble(), d['M'].toString().toDouble(), 3)
blng = baidu.x * 10000000000L
blat = baidu.y * 10000000000L
}
def estateType2 = "普通住宅"
if(name.contains("别墅")){
if (name.contains("别墅")) {
estateType2 = "别墅"
}else if(name.contains("商住")){
} else if (name.contains("商住")) {
estateType2 = "商住"
}else{
def type = d['I']
if(type){
if(type.contains("别墅")){
} else {
def type = d['J']
if (type) {
if (type.contains("别墅")) {
estateType2 = "别墅"
}else if(type.contains("商住")){
} else if (type.contains("商住")) {
estateType2 = "商住"
}
}
}
def key = city+"##"+newRegion+"##"+name
if(name && !keys.contains(key)){
def key = city + "##" + newRegion + "##" + name
if (name && !keys.contains(key)) {
def dist = CommonDatautils.findDistrict(city, newRegion, name, "district")
if(!dist){
if (!dist) {
keys << key
def data = new BasicDBObject();
data.put("name_cat",0)
data.put("status",0)
data.put("cat","district")
data.put("name_cat", 12)
data.put("status", 0)
data.put("cat", "district")
data.put("c_date", new Date())
data.put("m_date", new Date())
data.put("city", city)
data.put("region",newRegion)
data.put("region", newRegion)
data.put("name", name)
data.put("estate_type2",estateType2)
data.put("estate_type2", estateType2)
data.put("address", formatAddress)
if(glat){
data.put("lat",glat)
data.put("lng",glng)
data.put("lat2",blat)
data.put("lng2",blng)
if (glat) {
data.put("lat", glat)
data.put("lng", glng)
data.put("lat2", blat)
data.put("lng2", blng)
}
def markMap = [:]
markMap.put("data_source", "机器入库")
data.put("mark_map",markMap)
data.put("mark_map", markMap)
index++
DbUtils.seaweed.save(data);
println index+" "+data
println index + " " + data
def stop = false
while (!stop){
if(data._id.toString()){
while (!stop) {
if (data._id.toString()) {
addPrice(data)
stop = true;
}
@@ -213,19 +220,17 @@ class CcbDistrictToDB {
}
}
}
}else{
println city +" "+region +" 区域无法标准化"
}
},0);
}
def addPrice(dist){
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-03-01")
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-25")
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-05-01")
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-05-24")
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
for(def weekend : weekends){
@@ -435,9 +440,12 @@ class CcbDistrictToDB {
}
def nameFileter(def name){
name = name.replace("·","").replace(".","").replace("|","")
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
if(name){
name = name.replace("·","").replace(".","").replace("|","")
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
}
return name;
}
@@ -1,10 +1,12 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import org.apache.poi.ss.util.RegionUtil
import org.bson.types.ObjectId
import com.fangjia.core.util.BaiDuApiUtils
@@ -13,7 +15,82 @@ class DistrictMarge {
static main(args) {
def task = new DistrictMarge()
//task.run()
task.repeatAddress()
// task.repeatAddress()
// task.removeDist()
task.updateAddress()
// task.addMappingAlias()
}
def updateAddress(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\更新小区地址.xlsx", { d ->
if (d['A']) {
def region = d['B'].trim()
def newRegion = RegionData.formatRegion(d['A'], region)
if (newRegion) {
def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
if (dist) {
DBObject udata = new BasicDBObject()
udata.put("address", d['D'])
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set', udata))
}
}else{
println d['A'] + " 格式化失败 "+d['B']
}
}
});
}
def removeDist(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\删除小区.xlsx", { d ->
if (d['A']) {
def region = d['B']
def newRegion = RegionData.formatRegion(d['A'], region)
if (newRegion) {
def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
if (dist) {
DBObject udata = new BasicDBObject()
udata.put("status", 1)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set', udata))
}
}else{
println d['A'] + " 格式化失败 "+d['B']
}
}
})
}
def addMappingAlias(){
def index = 0
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\新增映射别名.xlsx", { d ->
index ++
if (d['A']) {
def region = d['B'].trim()
def newRegion = RegionData.formatRegion(d['A'], region)
if (newRegion) {
def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
if (dist) {
def mapping_alias = dist.mapping_alias
if(!dist.mapping_alias){
mapping_alias = []
}
def alias = d['D']
def arrs = alias.split("、")
for(def s : arrs){
if(!mapping_alias.contains(s)){
mapping_alias << s
}
}
DBObject udata = new BasicDBObject()
udata.put("mapping_alias", mapping_alias)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set', udata))
}
}else{
println d['A'] + " 格式化失败 "+d['B']
}
}
})
}
def repeatAddress(){
@@ -36,10 +36,37 @@ class MatchMapData {
// task.run3()
// task.exportHouse()
// task.addDistrictLevel()
task.run4()
// task.run4()
task.run5()
}
def run5(){
ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\½¨ÐÐ\\Êý¾ÝºËʵ.xlsx", { d ->
if (d['A']) {
def region = d['B']
def newRegion = RegionData.formatRegion(d['A'], region)
if (newRegion) {
def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
if(dist){
if(dist.mapping_alias){
d['D'] = dist.mapping_alias.join(",")
}
d['E'] = dist.address
}
def dist1 = CommonDatautils.findDistrict(d['A'], newRegion, d['G'], "district")
if(dist1){
if(dist1.mapping_alias){
d['H'] = dist1.mapping_alias.join(",")
}
d['I'] = dist1.address
}
}
}
});
}
def run4() {
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\核实小区是否存在.xlsx", { d ->
def regionCode = null
@@ -0,0 +1,108 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.ElasticUtils
import com.mongodb.BasicDBObject
import groovy.sql.Sql
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class OracleHouse {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
public static EsDataSource dataSource = new EsDataSource()
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
connectionProperties.put("user", "developer")
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
dataSource.setProperties(connectionProperties)
}
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
static main(args) {
def task = new OracleHouse()
task.run()
}
def run(){
def rets = findAllHouse("2022-04-19")
def totalCount = new AtomicLong()
println rets.size()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
def arrs = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def flag = findHouseFlag(arrs[0])
println totalCount.get()+" "+ arrs[0] +" "+flag
if(flag){
def dist = CommonDatautils.findDistrictById(arrs[1])
if(dist && dist.estate_type2=="ÆÕͨסլ"){
removeHouse(arrs[0])
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
}
def findHouseFlag(def id){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("_id", id))
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "district_comp_potential", queryBuilder);
def delFlag = false
for(def h : houses){
if(h.estate_type && (h.estate_type.contains("ÉÌ") || h.estate_type.contains("°ì"))){
println id+" "+h.estate_type
delFlag = true
}
}
return delFlag
}
def findAllHouse(def publishDate) {
def sql = "select FORSALEID,COMMID from bigdata_forsale_gj where publishtime>=to_date('${publishDate}','yyyy-MM-dd')";
def allRows = []
def index = 0
sqlInstance.eachRow(sql, { r ->
index = index + 1
println index
def data = []
data << r.FORSALEID
data << r.COMMID
allRows << data
});
return allRows
}
def removeHouse(houseid){
def sql = "delete from bigdata_forsale_gj where FORSALEID='${houseid}' ";
sqlInstance.executeUpdate(sql)
}
}
@@ -2,39 +2,70 @@ package com.fangjia.script.chongqing
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.DateUtil
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.gj.paimai.DataUtil
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.JacksonUtil
import com.fangjia.script.util.OpenApiClient
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.google.gson.JsonObject
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.sql.Sql
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.CopyOnWriteArrayList
import java.util.zip.GZIPInputStream
class MatchEvaluateRegionTest {
static def targetUrl = "http://192.168.0.12:9200/district_comp_potential/"
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def path = "D:\\数据\\美团\\10城小区明细-V10.xlsx"
def total = 1500
static main(args) {
def task = new MatchEvaluateRegionTest()
task.run()//小区匹配
task.run1()//地图匹配
// task.run()//小区匹配
// task.run1()//地图匹配
// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除
task.addPrice()
task.addAuthor()
// task.addPrice()
// task.addArroundPrice()
// task.addArroundPriceByLat()
// task.addRentRate()
// task.addAuthor()
// task.addLevel()
// task.addSorce()
// task.addArroundScore()
task.pkPrice()
}
def run(){
def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx"
def index = 0
ExcelUtils.multiThreadingModify(path, { row ->
index ++
if(!row['F']){
def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], "property", false, null, null)
println index+" "+row['E']
def jsonObject = OpenApiClient.addressMatch(row['A'], row['C'], "property", false,row['B'],null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['F'] = jsonObject.get("id")
row['G'] = jsonObject.get("city")
@@ -43,18 +74,17 @@ class MatchEvaluateRegionTest {
row['J'] = jsonObject.get("address")
row['K'] = jsonObject.get("propertyType")
row['L'] = jsonObject.get("map")
// row['K'] = jsonObject.get("lng")
// row['L'] = jsonObject.get("lat")
row['M'] = jsonObject.get("lng")
row['N'] = jsonObject.get("lat")
}
}
});
}
def run1(){
def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx"
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['F']){
def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], null, true, null, null)
if(!row['F'] && row['A']){
def jsonObject = OpenApiClient.addressMatch(row['A'], row['E'], null, true, row['B'],row['C'].toString())
if(jsonObject != null && jsonObject.precise!=-1) {
row['F'] = jsonObject.get("id")
row['G'] = jsonObject.get("city")
@@ -63,52 +93,104 @@ class MatchEvaluateRegionTest {
row['J'] = jsonObject.get("address")
row['K'] = jsonObject.get("propertyType")
row['L'] = jsonObject.get("map")
// row['K'] = jsonObject.get("lng")
// row['L'] = jsonObject.get("lat")
row['M'] = jsonObject.get("lng")
row['N'] = jsonObject.get("lat")
}
}
});
}
def pkPrice(){
ExcelUtils.multiThreadingModify(path, { row ->
if(row['R'] && row['R'].toString().toDouble().toInteger()==1){
def obj = CommonDatautils.queryDistrictPrice(row['G'], row['H'], row['I'], DateUtils.parseFromTightByPattern("2022-05-22","yyyy-MM-dd"))
if(!obj){
row['R'] = ''
row['S'] = "无最新价"
}
}
});
}
def addPrice(){
def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx"
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['L']){
def priceObj = CommonDatautils.findDistrictPrice(row['G'], row['H'], row['I'])
if(priceObj){
row['M'] = priceObj.avg_price
row['N']= (int)(priceObj.avg_price * row['E'])
row['O'] = priceObj.avg_price
// row['N']= (int)(priceObj.avg_price * row['E'])
}
def avg_price = findAvgPrice(row['M'])
if(row['M'] && avg_price){
def weekendPrice = row['M'].toString().toDouble()
def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
row['O'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15)
def sqmetreRentPrice = CommonDatautils.findNewSqmetreRent(row['G'], row['H'], row['I'])
if(sqmetreRentPrice){
row['P'] = sqmetreRentPrice
}
// def avg_price = findAvgPrice(row['F'])
// if(row['M'] && avg_price){
// def weekendPrice = row['M'].toString().toDouble()
// def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
// row['O'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15)
// }
}
});
}
def addRentRate(){
def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"]
def radioMap = [:]
for(def city : citys) {
def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1));
def avg_radio = CommonDatautils.findCityAvgRentRadio(city, lastMonthDay)
radioMap.put(city, avg_radio)
}
ExcelUtils.multiThreadingModify(path, { row ->
def city = row['G']
if (row['F'] && row['O'] && !row['P']) {
def avg_rent_radio = radioMap.get(city)
if(avg_rent_radio){
row['P'] = (int)(row['O']/avg_rent_radio)
row['S'] = "租金推算1"
}
}
})
}
def addArroundPrice(){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['O']) {
def arroundObj = FangjiaClientUtils.arroundPrice(row['A'], row['E'])
if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
def priceObj = arroundObj.result
row['O'] = priceObj.unitPrice
// row['N'] = (int) (priceObj.unitPrice * row['E'])
row['S'] = "周边价格"
}
}
});
}
def addAuthor(){
def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx"
def authors = ["王蕾","丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"]
def total = 747
def authors = ["丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"]
def personTotal = authors.size()
def avg = Math.ceil(total/personTotal)
def index = 0
def count = 0;
ExcelUtils.modify(path, { row ->
if(row['B']){
if(index < avg){
row['P'] = authors[count]
row['N'] = authors[count]
}
index++
if(index >= avg){
count++
index = 0
}
}
});
}
@@ -236,4 +318,295 @@ class MatchEvaluateRegionTest {
});
return avg_price
}
def addSorce(){
ExcelUtils.multiThreadingModify(path, { d ->
if(d['F']){
def score = findDistrictScore(d['F'])
if(score){
d['R'] = score
}
}
});
}
def addArroundScore(){
ExcelUtils.multiThreadingModify(path, { d ->
if(!d['R'] && d['F']){
def dist = null
if(d['F'].toString().size()>16){
dist = CommonDatautils.findDistrictById(d['F'])
}
if(!dist){
dist = new BasicDBObject()
dist.city = d['G']
dist.lng = d['M'].toString().toDouble()*10000000000L
dist.lat = d['N'].toString().toDouble()*10000000000L
}
if(dist.lat){
List<DBObject> list = DbUtils.findAroundDistrict(dist)
if (list) {
list = sortByDistance(dist, list);
if(list && list.size()>10){
list = list[0..9]
}
if(list){
def scores = []
for(def s : list){
def score = findDistrictScore(s._id.toString())
if(score){
scores << score
}
}
def arround_score =(int)(scores.sum()/scores.size())
d['R'] = arround_score
d['T'] = "周边小区评分2"
}
}
}
}
});
}
def addArroundPriceByLat(){
ExcelUtils.multiThreadingModify(path, { d ->
if(!d['O'] && d['F'] && (d['K'].contains("住宅") || d['K'].contains("商住") ||d['K'].contains("别墅"))){
def dist = null
if(d['F'].toString().size()>16){
dist = CommonDatautils.findDistrictById(d['F'])
}
if(!dist){
dist = new BasicDBObject()
dist.city = d['G']
dist.lng = d['M'].toString().toDouble()*10000000000L
dist.lat = d['N'].toString().toDouble()*10000000000L
}
if(dist.lat){
List<DBObject> list = DbUtils.findAroundDistrict(dist)
if (list) {
list = sortByDistance(dist, list);
if(list && list.size()>10){
list = list[0..9]
}
if(list){
def prices = []
for(def s : list){
def priceObj = CommonDatautils.findDistrictPrice(s.city, s.region, s.name)
if(priceObj){
prices << priceObj.avg_price
}
}
if(prices){
def arround_price =(int)(prices.sum()/prices.size())
d['O'] = arround_price
d['S'] = "周边均价3"
}
}
}
}
}
});
}
def sortByDistance(d, list){
def newList = []
for(def nb : list){
def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0)
nb.distance = distance
newList << nb
}
//排序
Collections.sort(newList, new Comparator() {
@Override
int compare(Object o1, Object o2) {
if(o1.distance > o2.distance)
return 1;//第二个元素(o1)比第一个元素(o2)大,返回-1
if(o1.distance == o2.distance)
return 0;
return -1;
}
})
return newList;
}
def findDistrictScore(id){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("_id", id))
def indexObj = ElasticUtils.executeSimpleQuery("192.168.0.12", "district_comp_potential", queryBuilder);
def score = 0
for(def h : indexObj){
score = h.total_price_rate
if(score){
break;
}
}
return score
}
def addLevel() {
def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"]//
def weekend = DateUtils.getFirstDayOfWeek(new Date());
def cityLevelMap = [:]
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region","name"])
def priceMap = CommonDatautils.queryDistrictPrice(city, weekend)
def datas = addPriceData(dists, priceMap);
datas = calcuteLevel(datas);
cityLevelMap.put(city, ["price":priceMap, "level":datas])
println city +" 楼盘评级同步完成"
}
ExcelUtils.multiThreadingModify(path, { row ->
def address = row['E']
def city = RegionData.formatCity(row['A'])
if(row['K'] && !row['K'].contains("住宅") && !row['K'].contains("商住") && !row['K'].contains("别墅")){
if(city){
def map = cityLevelMap.get(city)
def rMap = null
if(map){
rMap = getArroundDistrictPriceByAddress(city, address, map.level)
}
if(rMap){
if(rMap.rank){
row['Q'] = rMap.rank
}
}
}
}
})
}
def addPriceData(dists, priceMap){
def newLists = []
for(def d : dists){
def price = priceMap.get(d.city+"##"+d.region+"##"+d.name);
if(price){
d.avg_price = price;
newLists << d;
}
}
if(newLists){
newLists.sort {a,b->
def v1 = a.avg_price
def v2 = b.avg_price
return v1.compareTo(v2)
}
}
return newLists;
}
def calcuteLevel(def datas){
def total = datas.size()
def v = (int)(total/10)
def start = 0;
def end = 0;
def map = [:]
for(int i=1;i<=10;i++){
end = end + v;
if(i == 10){
map.put(i, [start, total])
}else{
map.put(i, [start, end])
}
start = end;
}
for(def d=0;d<datas.size();d++){
def dist = datas[d]
for (def o in map) {
def ranges = o.value
def m = ranges[0]
def n = ranges[1]
if(d==0){
if(d<=n){
dist.level = 10 - o.key + 1
break
}
}else{
if(d>m && d<=n){
dist.level = 10 - o.key + 1
break
}
}
}
}
return datas;
}
def getArroundDistrictPriceByAddress(def city, def address, def levels){
def json = GaoDeApiUtils.geo(city, address)
def location = null
def detail = new BasicDBObject()
def level = null
def rank = null
if(json && json.count>=1){
def m = json.geocodes[0]
if(m){
location = m.location
level = m.level
}
}
if(location){
def arrs = location.split(",")
detail.lng = arrs[0].toString().toDouble()*10000000000L
detail.lat = arrs[1].toString().toDouble()*10000000000L
}
if(detail.lng){
detail.city = city
List<DBObject> list = DbUtils.findAroundDistrictByRadius(detail, 3000)
if(list && list.size()>10){
list = sortByDistance(detail, list);
try{
list = list[0..9]
}catch(Exception e){
println list.size()
}
}
def arroundAvgPrice = getArroundAvgPrice(list)
if(arroundAvgPrice){
def mod = Integer.MAX_VALUE;
for(def d : levels){
def newMod = java.lang.Math.abs( d.avg_price - arroundAvgPrice)
if(newMod<mod){
mod = newMod
rank = d.level
}
}
}
}
def m = [:]
m.level = level
m.rank = rank
return m
}
def getArroundAvgPrice(def list){
def prices = []
for(def l : list){
def priceObj = CommonDatautils.findDistrictPrice(l.city, l.region, l.name)
if(priceObj){
prices << priceObj.avg_price
}
}
if(prices){
def avg_price = (prices.sum()/prices.size()).toInteger()
return avg_price;
}
return null
}
}
@@ -28,14 +28,14 @@ class MatchEvaluateTest {
static main(args) {
def task = new MatchEvaluateTest()
// task.run()//小区匹配
// task.run1()//地图匹配
//// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除
// task.addPrice()
// task.addArroundPrice()
// task.addAuthor()
task.run()//小区匹配
task.run1()//地图匹配
// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除
task.addPrice()
task.addArroundPrice()
task.addAuthor()
task.stat()
// task.stat()
}
def addArroundPrice(){
@@ -57,7 +57,7 @@ class MatchEvaluateTest {
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['D'] && row['B']){
def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null)
def jsonObject = OpenApiClient.addressMatch5("重庆", row['B'], "property", false)
if(jsonObject != null && jsonObject.precise!=-1) {
row['D'] = jsonObject.get("id")
row['E'] = jsonObject.get("city")
@@ -77,7 +77,7 @@ class MatchEvaluateTest {
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['D'] && row['B']){
def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], null, true, null, null)
def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], null, true)
if(jsonObject != null && jsonObject.precise!=-1) {
row['D'] = jsonObject.get("id")
row['E'] = jsonObject.get("city")
@@ -117,7 +117,7 @@ class MatchEvaluateTest {
def addAuthor(){
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
def authors = ["王蕾","丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"]
def authors = ["丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"]
def personTotal = authors.size()
def avg = Math.ceil(total/personTotal)
@@ -0,0 +1,666 @@
package com.fangjia.script.chongqing
import com.fangjia.core.util.SourceNameUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import groovy.sql.Sql
import org.bson.types.ObjectId
import java.util.concurrent.CopyOnWriteArrayList
class PocTest {
def total = 3259
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new PocTest();
task.run();
}
/**
* A 序号
* B 城市
* C 区域
* D 地址
* E 面积
* @return
*/
def run(){
def inputfile = "D:\\数据\\重庆农商行\\poc\\test.xls"
// match(inputfile)
// addPrice(inputfile)
// addArroundPrice(inputfile)
// standard(inputfile)
// addLoudongNum(inputfile)
// addAuthor(inputfile)
// pkPrice(inputfile)
// addPpeValue(inputfile)
// matchMap(inputfile)
// addDetail(inputfile, "D:\\数据\\重庆农商行\\poc\\楼盘详情.xlsx")
//
// createLoudong(inputfile,"D:\\数据\\重庆农商行\\poc\\底层数据-楼栋-户.xlsx")
// createPaimai(inputfile,"D:\\数据\\重庆农商行\\poc\\法拍案例.xlsx")
// createHouse(inputfile,"D:\\数据\\重庆农商行\\poc\\挂牌成交评估案例.xlsx")
// createDealPrice(inputfile,"D:\\数据\\重庆农商行\\poc\\挂牌成交评估案例.xlsx")
// createEvaluation(inputfile,"D:\\数据\\重庆农商行\\poc\\挂牌成交评估案例.xlsx")
}
def addPpeValue(inputfile){
ExcelUtils.multiThreadingModify(inputfile, { row ->
if(row['F'] && row['R']){
def avg_price = findAvgPrice(row['R'])
if(avg_price){
def ppe15 = Math.abs((row['F'] - avg_price) * 100 / avg_price)
row['Z'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15)
}
}
});
}
def pkPrice(path){
ExcelUtils.multiThreadingModify(path, { row ->
def commid = row['R']
if(commid){
def dist = CommonDatautils.findDistrictById(commid.trim())
if(dist){
def jsonObject = OpenApiClient.webankEvaluate(dist.city, dist.region, dist.name, row['E'].toString())
if(jsonObject != null && jsonObject.code==200) {
def result = jsonObject.result
if(result){
row['Y'] = result.get("unitPrice")
}
}
}
}
})
}
def match(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['Q'] && row['D']){
def city = row['B']
def address = row['D']
def region = row['C']
println "[match] city:"+city+" region:"+region+" address:"+address
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['Q'] = jsonObject.get("name")
row['R'] = jsonObject.get("id")
}
}
})
}
def matchMap(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['Q'] && row['D']){
def city = row['B']
def address = row['D']
def region = row['C']
println "city:"+city+" region:"+region+" address:"+address
def jsonObject = OpenApiClient.addressMatch(city, address, null, true, region, null)
if(jsonObject != null && jsonObject.precise!=-1) {
def propertyType = jsonObject.get("propertyType")
if(propertyType && (propertyType.contains("住宅") || propertyType.contains("别墅") || propertyType.contains("商住"))){
row['Q'] = jsonObject.get("name")
row['R'] = new ObjectId().toString()
row['W'] = propertyType
def arroundObj = FangjiaClientUtils.arroundPrice(city, address)
if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
def priceObj = arroundObj.result
row['F'] = priceObj.unitPrice
}
}
}
}
})
}
def addPrice(path){
ExcelUtils.multiThreadingModify(path, { row ->
def commid = row['R']
if(commid){
def dist = CommonDatautils.findDistrictById(commid.trim())
if(dist){
row['Q'] = dist.name
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
if(priceObj){
row['F'] = priceObj.avg_price
}
}
}
})
}
def addArroundPrice(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['F']) {
def arroundObj = FangjiaClientUtils.arroundPrice(row['B'], row['D'])
if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
def priceObj = arroundObj.result
row['F'] = priceObj.unitPrice
row['W'] = "周边均价"
}
}
});
}
def standard(path){
ExcelUtils.multiThreadingModify(path, { row ->
def city = row['B']
def address = row['D']
if(address){
println "[standard] address:"+address
def jsonObject = OpenApiClient.standard2Action(city, address)
if(jsonObject){
println jsonObject
row['G'] = jsonObject.get("province")
row['H'] = jsonObject.get("city")
row['I'] = jsonObject.get("district")
row['J'] = jsonObject.get("township")
row['K'] = jsonObject.get("street")
row['L'] = jsonObject.get("streetNumber")
row['M'] = jsonObject.get("buildingNumber")
row['N'] = jsonObject.get("unitNumber")
row['O'] = jsonObject.get("floor")
row['P'] = jsonObject.get("roomNumber")
}
}
})
}
def addLoudongNum(path){
ExcelUtils.multiThreadingModify(path, { row ->
def commid = row['R']
def houseNum = row['M']
def unitNum = row['N']
println "[addLoudongNum] commid:"+commid+" houseNum:"+houseNum
if (commid && houseNum) {
def jsonObject = OpenApiClient.searchLoudong(commid, houseNum, unitNum)
if (jsonObject && jsonObject.result) {
if(jsonObject.result.size()>=1){
def obj = jsonObject.result[0]
row['T'] = obj.id
row['S'] = obj.houseNumber
}
}
if(row['P']){
def jsonObject1 = OpenApiClient.searchHouseLoudong(commid, houseNum, unitNum, row['P'])
if (jsonObject1 && jsonObject1.result) {
if(jsonObject1.result.size()>=1){
def obj = jsonObject1.result[0]
row['V'] = obj.id
row['U'] = obj.roomNum
}
}
}
}
})
}
def createLoudong(path, output){
def rets = new CopyOnWriteArrayList()
def datas = ExcelUtils.read(path, 0, 1)
rets << ["输入城市","输入区域","输入地址","面积","楼栋号","楼栋ID","物业类型","主朝向","地上层数","地下层数","竣工日期","梯","户","总户数","景观","户号","户ID","主朝向","景观","户型结构"]
for(def row : datas){
def data = []
data << row['B']
data << row['C']
data << row['D']
data << row['E']
if (row['R'] && row['M']) {
def jsonObject = OpenApiClient.searchLoudong(row['R'], row['M'], row['N'])
if (jsonObject && jsonObject.result && jsonObject.result.size()>=1) {
def obj = jsonObject.result[0]
data << obj.houseNumber
data << obj.id
data << obj.type// 物业类型
data << ""//主朝向
data << obj.floors//地上层数
data << ""//地下层数
data << obj.buildYear;//竣工日期
data << ""//梯
data << ""//户
data << obj.households//总户数
data << ""//景观
}else{
data << ""
data << ""
data << ""// 物业类型
data << ""//主朝向
data << ""//地上层数
data << ""//地下层数
data << ""//竣工日期
data << ""//梯
data << ""//户
data << ""//总户数
data << ""//景观
}
if(row['P']){
def jsonObject1 = OpenApiClient.searchHouseLoudong(row['R'], row['M'], row['N'], row['P'])
if (jsonObject1 && jsonObject1.result) {
if(jsonObject1.result.size()>=1){
def obj = jsonObject1.result[0]
data << obj.roomNum//户号
data << obj.id//户ID
data << obj.direction//主
data << obj.sight//景观
if(obj.roomNum && obj.roomNum.contains("跃")){
data << "跃式"//户型结构
}else{
data << ""//户型结构
}
}
}
}
}
rets << data
}
ExcelUtils.write(output, rets, "底层数据-楼栋-户")
}
def addDetail(path, output){
def datas = ExcelUtils.read(path, 0, 1)
def rows = []
rows << ["输入城市","输入区域","输入地址","小区名称","小区ID","小区别名","小区地址","板块","街道","基准价","经纬度","竣工时间","开发商名称","绿化率","容积率","车位个数","物业公司","物业费","住宅幢数","总户数","车位配比","占地面积","是否学区","周边教育","周边交通","周边商业","周边医院","周边金融机构"]
for(def d : datas){
def commid = d['R']
println "[addDetail] commid:"+commid
if(commid){
def dist = CommonDatautils.findDistrictById(commid)
if(dist){
def row = []
def jsonObject = OpenApiClient.districtDetail(dist.city, dist.region, dist.name)
if(jsonObject) {
row << d['B']
row << d['C']
row << d['D']
row << jsonObject.get("name")//小区名称
row << commid//小区ID
if(jsonObject.get("alias")){
row << jsonObject.get("alias").join(",").replace('"',"")//小区别名
}else{
row << ""
}
row << jsonObject.get("address")//小区地址
row << jsonObject.get("commercialArea")//板块
row << jsonObject.get("township")//街道***********
row << jsonObject.get("avgPrice")//基准价
row << jsonObject.get("lng").toString()+","+jsonObject.get("lat").toString()//经纬度
row << jsonObject.get("finishBuildTime")//竣工时间
row << jsonObject.get("developer")//开发商名称
row << SeaweedUtils.formatVirescencePercent(jsonObject.get("greenPercentage"))//绿化率
row << SeaweedUtils.formatCubagePercent(jsonObject.get("plotRatio"))//容积率
row << SeaweedUtils.formatCarSiteCount(jsonObject.get("parking"))//车位个数
row << jsonObject.get("propertyManagement")//物业公司
row << jsonObject.get("managementFee")//物业费
row << jsonObject.get("buildingCount")//住宅幢数
row << SeaweedUtils.formatHouseholdCount(jsonObject.get("totalUnitCount"))//总户数
row << SeaweedUtils.calcCarAndHouseholdRadio(jsonObject.get("parking"), jsonObject.get("totalUnitCount"))//车位配比*********
row << SeaweedUtils.formatAllArea(jsonObject.get("floorSpace"))//占地面积
}
def facilityObj = OpenApiClient.facilitys(dist.city, dist.region, dist.name)
if(facilityObj && facilityObj.result){
def result = facilityObj.result
if(result.keySchool){
row << '是'
}else {
row << ""
}
if(result.school){
row << result.school.join(",").replace('"',"")
}else {
row << ""
}
def bus = []
if(result.bus){
bus.addAll(result.bus)
}
if(result.metro){
bus.addAll(result.metro)
}
if(bus){
row << bus.join(",").replace('"',"")
}else{
row << ""
}
if(result.supermarket){
row << result.supermarket.join(",").replace('"',"")
}else{
row << ""
}
if(result.hospital){
row << result.hospital.join(",").replace('"',"")
}else{
row << ""
}
if(result.bank){
row << result.bank.join(",").replace('"',"")
}else{
row << ""
}
}
rows << row
}
}
}
ExcelUtils.write(output, rows, "底层数据-小区")
}
def createPaimai(path, output){
def datas = ExcelUtils.read(path, 0, 1)
def rows = []
rows << ["输入城市","输入区域","输入地址","小区名称","标的名称","楼栋","户号","面积","案例时间","类型","状态","成交单价","评估单价"]
def keys = []
for(def d : datas){
def commid = d['R']
println "[createPaimai] commid:"+commid
if(commid) {
def dist = CommonDatautils.findDistrictById(commid)
if (dist) {
def key = dist.city+"#"+dist.region+"#"+dist.name
if(keys.contains(key)){
continue
}
keys << key
def jsonObject = OpenApiClient.searchPaimais(dist.city, dist.region, dist.name,"20220217",100)
if(jsonObject && jsonObject.result){
def results = jsonObject.result
for(def r: results){
def row = []
row << d['B']
row << d['C']
row << d['D']
row << r.matchName
row << r.address
row << r.houseNum
row << r.roomNum
row << r.area
if(r.startDate){
row << r.startDate.substring(0, r.startDate.indexOf("T"))
}else{
row << ""
}
row << r.count
row << r.status
row << r.tradeAvgPrice
row << r.tradePrice
row << r.id
rows << row
}
println dist.name +" "+results.size()
}
}
}
}
ExcelUtils.write(output, rows, "拍卖案例")
}
def createHouse(path, output){
def datas = ExcelUtils.read(path, 0, 1)
def rows = []
rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
def keys = []
for(def d : datas) {
def commid = d['R']
// println "[createHouse] commid:"+commid
if (commid) {
def key = d['Q']
if (keys.contains(key)) {
continue
}
keys << key
def jsonObject = OpenApiClient.houseList(commid,"20210501",20)
if(jsonObject && jsonObject.result) {
def results = jsonObject.result
for (def r : results) {
def row = []
row << "挂牌"
row << r.districtName
if(r.estateType){
row << r.estateType
}else{
if(r.purpose){
row << r.purpose
}else{
row << ""
}
}
if(r.room){
def roomStr = r.room+"室"
if(r.hall){
roomStr = roomStr +r.hall+"厅"
}
row << roomStr
}else{
row << ""
}
row << r.area
row << r.fitment
row << r.pDate
row << ""
row << r.avgPrice
row << r.totalPrice
if(r.floor){
row << r.floor
}else{
row << ""
}
if(r.height){
row << r.height
}else{
row << ""
}
row << r.direction
row << SourceNameUtil.getResourceName(r.root)
rows << row
}
println d['R'] +" "+results.size()
}
}
}
ExcelUtils.write(output, rows, "挂牌案例")
}
def createEvaluation(path, output){
def datas = ExcelUtils.read(path, 0, 1)
def rows = []
rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
def keys = []
for(def d : datas) {
def commid= d['R']
println "[createEvaluation] commid:"+commid
if (commid) {
if (keys.contains(commid)) {
continue
}
keys << commid
def jsonObject = OpenApiClient.evaluaionList(commid,"20210501",20)
if(jsonObject && jsonObject.result) {
def results = jsonObject.result
for (def r : results) {
def row = []
row << "评估"
row << r.commName
row << ""
if(r.roomcnt){
def roomStr = r.roomcnt+"室"
if(r.livingroomcnt){
roomStr = roomStr +r.livingroomcnt+"厅"
}
row << roomStr
}else{
row << ""
}
row << r.buildArea
row << r.fitment
row << r.cDate.substring(0, r.cDate.indexOf("T"))
row << ""
row << r.unitPrice
row << r.totalPrice
if(r.floor){
row << r.floor
}else{
row << ""
}
if(r.floorcnt){
row << r.floorcnt
}else{
row << ""
}
if(r.houseaspect && !r.houseaspect.contains("暂")){
row << r.houseaspect
}else{
row << ""
}
row << "房价网"
rows << row
}
println d['R'] +" "+results.size()
}
}
}
ExcelUtils.write(output, rows, "评估案例")
}
def createDealPrice(path, output){
def datas = ExcelUtils.read(path, 0, 1)
def rows = []
rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
def keys = []
for(def d : datas) {
def commid= d['R']
println "[createDealPrice] commid:"+commid
if (commid) {
if (keys.contains(commid)) {
continue
}
keys << commid
def jsonObject = OpenApiClient.searchDealPrice(commid,"20210501",20)
if(jsonObject && jsonObject.result) {
def results = jsonObject.result
for (def r : results) {
def row = []
row << "成交"
row << r.districtName
if(r.type){
row << r.type
}else{
row << ""
}
if(r.room){
def roomStr = r.room+"室"
if(r.hall){
roomStr = roomStr +r.hall+"厅"
}
row << roomStr
}else{
row << ""
}
row << r.area
row << r.fitment
row << r.tradeDate.substring(0, r.tradeDate.indexOf("T"))
row << ""
row << r.avgPrice
row << r.totalPrice
if(r.floor){
row << r.floor
}else{
row << ""
}
if(r.height){
row << r.height
}else{
row << ""
}
if(r.direction && !r.direction.contains("暂")){
row << r.direction
}else{
row << ""
}
row << r.source
rows << row
}
println d['R'] +" "+results.size()
}
}
}
ExcelUtils.write(output, rows, "成交案例")
}
def addAuthor(def path){
def authors = ["丁世菁","王文艳","董彦均","曹玉彦","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","穆良书","沈燕芳"]
def personTotal = authors.size()
def avg = Math.ceil(total/personTotal)
def index = 0
def count = 0;
ExcelUtils.modify(path, { row ->
if(row['B']){
if(index < avg){
row['X'] = authors[count]
}
index++
if(index >= avg){
count++
index = 0
}
}
});
}
def findAvgPrice(def commid){
def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2";
def avg_price = null
sqlInstance.eachRow(sql, { r ->
avg_price = r.target_avg_price
});
return avg_price
}
}
@@ -68,6 +68,10 @@ class CommonDatautils {
public static final DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName2("fangjia", "sqmetre_rent", mongoDbUrl)
public static final DBCollection LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia_base", "loudong", mongoDbUrl)
public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", mongoDbUrl)
public static DSCollection getHouseDataStoreConnection(){
if(house_dataStoreConnection != null) {
return house_dataStoreConnection
@@ -586,8 +590,8 @@ class CommonDatautils {
}
public static def findDistrict(String city, String region, String name, String cat){
public static def findDistrictCount(String city, String region, String name, String cat){
DBObject q = new BasicDBObject();
q.put("city", city);
if(region){
@@ -598,7 +602,24 @@ class CommonDatautils {
q.put("cat", cat)
q.put("status", 0)
def count = getDataStoreConnection().count(q);
return count;
}
public static def findDistrict(String city, String region, String name, String cat){
DBObject q = new BasicDBObject();
q.put("city", city);
if(region){
q.put("region", region);
}
q.put("name", name);
q.put("cat", cat)
q.put("status", 0)
DSCursor cursor = getDataStoreConnection().find(q);
if(cursor){
for(def obj : cursor){
@@ -609,6 +630,7 @@ class CommonDatautils {
}
public static def findDistrictList(String city, String region, String name, String cat){
DBObject q = new BasicDBObject();
@@ -704,7 +726,7 @@ class CommonDatautils {
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
// q.put("name","ÎâÖз9ÅªÐ¡Çø")
// q.put("name","É­Íû·¼²ÝÂ̰¶")
DBObject keys = new BasicDBObject();
if(fields){
@@ -827,16 +849,24 @@ class CommonDatautils {
keys.put("out_pics", 1);
keys.put("lat", 1);
keys.put("lng", 1);
keys.put("lat2", 1);
keys.put("lng2", 1);
keys.put("name_cat", 1);
keys.put("finish_building_date", 1)
keys.put("mapping_alias", 1)
keys.put("sid", 1)
DSCursor cursor = getDataStoreConnection().find(q, keys);
if(cursor){
for(def obj : cursor){
return obj;
try{
DSCursor cursor = getDataStoreConnection().find(q, keys);
if(cursor){
for(def obj : cursor){
return obj;
}
}
}catch(Exception e){
println "error_id:"+id
e.printStackTrace()
}
return null;
}
@@ -1053,9 +1083,16 @@ class CommonDatautils {
public static def queryDistrictById(def id){
def dbo = null
DBObject q = new BasicDBObject();
q.put("_id", new ObjectId(id));
def dbo = getDataStoreConnection().findOne(q);
try{
q.put("_id", new ObjectId(id.trim().toString()));
dbo = getDataStoreConnection().findOne(q);
}catch(Exception e){
println id
}
return dbo;
}
@@ -2385,6 +2422,7 @@ class CommonDatautils {
q.put("block", block);
}
q.put("cat", cat);
q.put("status",0)
def cur = getDataStoreConnection().find(q);
return cur.count();
}
@@ -3041,18 +3079,26 @@ class CommonDatautils {
queryCondition.put("region", region);
queryCondition.put("district_name", name);
queryCondition.put('weekend', new BasicDBObject('$gte', startDate).append('$lte', endDate));
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1)).limit(1)
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
DBObject dbo;
def prices = []
def index = 0
while(cur.hasNext()) {
dbo = cur.next()
def price = dbo.get("avg_price");
if(price){
break;
def obj = cur.next()
if(index == 0){
dbo = obj
}
prices << obj.get("avg_price");
index++
}
return dbo;
if(prices){
def avg_price = (int)(prices.sum()/prices.size())
dbo.avg_price = avg_price
return dbo;
}
return null;
}
@@ -3184,5 +3230,19 @@ class CommonDatautils {
def avg_radios = (int)(radios.sum()/radios.size())
return avg_radios;
}
static def findLoudong(commid){
def q = new BasicDBObject()
q.put("id_in_seaweed", commid)
def count = LOUDONG_LINE_COLL.count(q)
return count;
}
static def findHouseLoudong(commid){
def q = new BasicDBObject()
q.put("district_id", commid)
def count = HOUSE_LOUDONG_COLL.count(q)
return count;
}
}
@@ -35,7 +35,7 @@ class DistrictAroundFacilitiesBatch extends BaseBatch {
@Override
public void process(Object args) {
def cities = ["北京", "上海"]
def cities = ["北京", "上海"]//
for (def city : cities) {
def ret = CommonDatautils.getAllDistricts("district", city)
// def ret = getAllDistrictBaseInfo("上海", '闵行', '万源城', "district");
@@ -6,10 +6,16 @@ import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.jusfoun.JusfounCommon
import com.fangjia.script.util.DateUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.codec.digest.DigestUtils
import org.apache.commons.io.FileUtils
import org.bson.types.ObjectId
import java.text.DecimalFormat
import java.util.concurrent.TimeUnit
/**
@@ -19,8 +25,11 @@ import java.util.concurrent.TimeUnit
class DistrictPriceExportBatch extends BaseBatch{
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def lastMonthDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
def charset = "GBK"
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
static main(args) {
runBatch(new DistrictPriceExportBatch(), args)
}
@@ -32,6 +41,9 @@ class DistrictPriceExportBatch extends BaseBatch{
def ret = createDatas("¸£ÖÝ")
saveToDB(ret)
def fileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".csv"
def dataCodeFileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".sha512"
if(CommonDatautils.isWindow()){
@@ -76,10 +88,10 @@ class DistrictPriceExportBatch extends BaseBatch{
def createDatas(def city){
def districts = CommonDatautils.findAllDistricts(city, "district", null);
def lastDate = new Date()
def startDate = DateUtils.getFirstDayOfMonth(lastDate, -1)
def endDateStr = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-10"
def dataStr = DateUtils.add(endDateStr, -30)
def startDate = DateUtils.parseFromTightYYYYMMDD(dataStr)
def lastDate = DateUtils.parseFromTightByPattern(endDateStr, "yyyy-MM-dd")
// List<String> list2 = FileUtils.readLines(new File("D:\\Êý¾Ý\\¸£ÖÝ\\FJW_DISTRICT_STAT_202111.csv"))
// def map2 = [:]
// for(def s : list2) {
@@ -88,17 +100,34 @@ class DistrictPriceExportBatch extends BaseBatch{
// map2.put(key, row[5])
// }
def lastMonthPriceMap = findFzDistrictStat("¸£ÖÝ",lastMonthDate)
def result = []
for(def rMap : districts){
def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate)
if(dbo){
def lastMonthPrice = lastMonthPriceMap.get(rMap.get("_id").toString())
def curPrice = dbo.get("avg_price")
if(lastMonthPrice && curPrice){
def markup = (curPrice - lastMonthPrice)*100/lastMonthPrice
if(Math.abs(markup)>3){
def randMarkup = nextDouble(0, 3)
if(markup < 0){
markup = randMarkup*(-1)
}else{
markup = randMarkup
}
curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice)
}
}
def m = []
m << dbo.get("_id").toString()//¾ù¼ÛID
m << rMap.get("_id").toString()//Â¥ÅÌID
m << rMap.city
m << rMap.region
m << rMap.name
m << dbo.get("avg_price")
m << curPrice
// def lastMonthPrice = map2.get(rMap.city + "##" + rMap.region+"##"+rMap.name)
// if(lastMonthPrice && dbo.get("avg_price")){
// lastMonthPrice = lastMonthPrice.toInteger()
@@ -125,6 +154,19 @@ class DistrictPriceExportBatch extends BaseBatch{
return result
}
def findFzDistrictStat(city,lastMonthDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("month", lastMonthDate)
def dbcursor = FZ_DISTRICT_STAT_COLL.find(q)
def priceMap = [:]
for(def d : dbcursor){
priceMap.put(d.comm_id, d.avg_price)
}
return priceMap
}
def export(def fileName, def datas){
def headers = ["id","comm_id","city","district","name","avg_price","year_month"]
writeCSV(headers, fileName, datas);
@@ -152,5 +194,44 @@ class DistrictPriceExportBatch extends BaseBatch{
file.append(System.getProperty("line.separator"));
}
}
//Ëæ»ú¸¡µãÊý
public static double nextDouble(final double min, final double max) throws Exception {
DecimalFormat df = new DecimalFormat("#.00");
if (max < min) {
throw new Exception("min < max");
}
if (min == max) {
return min;
}
return Double.parseDouble(df.format(min + ((max - min) * new Random().nextDouble())));
}
def saveToDB(rets){
for(def r : rets){
def q = new BasicDBObject()
q.put("month", curDate.toInteger())
q.put("comm_id", r[1])
def obj = FZ_DISTRICT_STAT_COLL.findOne(q)
if(obj){
def udata = new BasicDBObject()
udata.put("m_date", new Date())
udata.put("avg_price", r[5])
FZ_DISTRICT_STAT_COLL.update(q, new BasicDBObject('$set', udata))
}else{
def udata = new BasicDBObject()
def o = new BasicDBObject()
o.put("_id", new ObjectId(r[0]))
o.put("comm_id", r[1])
o.put("city", r[2])
o.put("region", r[3])
o.put("name", r[4])
o.put("avg_price", r[5].toInteger())
o.put("month", r[6].toInteger())
FZ_DISTRICT_STAT_COLL.save(o)
}
}
}
}
@@ -8,7 +8,9 @@ import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBCursor
import com.mongodb.DBObject
import com.rabbitmq.client.impl.AMQImpl
import org.apache.commons.io.FileUtils
import org.bson.types.ObjectId
/**
* 福州数据结果统计
@@ -17,16 +19,42 @@ class FzDataResultStat {
public static final DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName("fangjia", "region_stat", "mongo.db.fangjia.com")
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
static main(args) {
def task = new FzDataResultStat()
task.priceMarkupRun();
task.regionPriceTrendRun()
//// task.margeExcel()
// task.saveCsvData()
}
def margeExcel(){
def list2= ExcelUtils.read("D:\\数据\\福州\\价格比对_202204_新2.xlsx", 0 ,1)
def map2 = [:]
for(def s : list2){
def key = s["A"]+"##"+s["B"]+"##"+s["C"]
map2.put(key, s["D"])
}
ExcelUtils.multiThreadingModify("D:\\数据\\福州\\价格比对_202204_新3.xlsx",{row->
if(row["A"]){
def key = row["A"]+"##"+row["B"]+"##"+row["C"]
def price = map2.get(key)
if(price){
row['L'] = price
}
}
})
}
def priceMarkupRun(){
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
def map1 = [:]
for(def s : list1){
def row = s.split("@\\|@")
@@ -137,16 +165,13 @@ class FzDataResultStat {
queryCondition.put('weekend', new BasicDBObject('$gte', startDate).append('$lte',endDate));
DBCursor cur = REGION_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def price = 0;
def prices = [];
while(cur.hasNext()) {
DBObject dbo = cur.next()
price = dbo.get("avg_price");
if(price){
break;
}
prices << dbo.get("avg_price");
}
return price;
def avg_price = (int)(prices.sum()/prices.size())
return avg_price;
}
@@ -165,4 +190,45 @@ class FzDataResultStat {
weekends = weekends.reverse();
return weekends;
}
def saveCsvData(){
def files = getAllFiles("C:\\Users\\gaojuan\\福州数据(1)")
if(files){
for(def filepath : files){
List<String> list1= FileUtils.readLines(filepath)
println filepath
for(def s : list1){
def row = s.split("@\\|@")
def o = new BasicDBObject()
o.put("_id", new ObjectId(row[0]))
o.put("comm_id", row[1])
o.put("city", row[2])
o.put("region", row[3])
o.put("name", row[4])
o.put("avg_price", row[5].toInteger())
o.put("month", row[6].toInteger())
FZ_DISTRICT_STAT_COLL.save(o)
}
}
}
}
def static getAllFiles(def path) {
File file = new File(path);
File[] tempList = file.listFiles();
def files = [];
for (int i = 0; i < tempList.length; i++) {
if (tempList[i].isFile()) {
files << tempList[i];
}
if (tempList[i].isDirectory()) {
System.out.println("文件夹:" + tempList[i]);
}
}
return files;
}
}
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202203.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202204.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -112,22 +112,28 @@ class TradeExportBatch extends BaseBatch{
// q.put("trade_date", new BasicDBObject(QueryOperators.LT, endDate))
def cursor = DEAL_PRICE.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(1000);
def rets = []
def keys = []
if (cursor != null) {
while (cursor.hasNext()) {
DBObject dbo = cursor.next();
def district = districtMap.get(dbo.city + "_" + dbo.region + "_" + dbo.district_name);
if(district){
def ret = [];
ret << dbo._id.toString()
ret << district._id.toString()
ret << dbo.city
ret << dbo.region
ret << dbo.district_name
ret << dbo.avg_price
ret << dbo.area
ret << DateUtils.formatDateByPattern(dbo.trade_date, "yyyy-MM-dd")
ret << dbo.total_price
rets << ret;
def tradeDateStr = DateUtils.formatDateByPattern(dbo.trade_date, "yyyy-MM-dd")
def key = district._id.toString() +"##" + dbo.avg_price +"##"+dbo.area+"##"+tradeDateStr
if(!keys.contains(key)){
keys << key
def ret = [];
ret << dbo._id.toString()
ret << district._id.toString()
ret << dbo.city
ret << dbo.region
ret << dbo.district_name
ret << dbo.avg_price
ret << dbo.area
ret << tradeDateStr
ret << dbo.total_price
rets << ret;
}
}
}
println "³É½»×ܼƣº"+rets.size()
@@ -35,7 +35,8 @@ class EsHousePriceStat extends BaseBatch{
def run(){
def cityMap = CommonDatautils.getAllCityRegions()
cityMap.each{city,regions->
findPriceByCity(city, regions);
findPriceByCity(city, regions);
}
}
@@ -64,9 +65,9 @@ class EsHousePriceStat extends BaseBatch{
q.put("region" , r.region)
q.put("month", r.month)
def mom = null
if(lastObj){
mom = (r.price -lastObj.avg_price)/lastObj.avg_price
mom = DataHandleUtils.getDoubleRound(4, mom)
if(lastObj && lastObj.avg_price && r.price){
mom = (r.price -lastObj.avg_price)/lastObj.avg_price
mom = DataHandleUtils.getDoubleRound(4, mom)
}
if(obj){
if(obj.avg_price != r.price){
@@ -103,7 +104,7 @@ class EsHousePriceStat extends BaseBatch{
def findEsAvgPrice(city, region){
def detail = [:]
detail.put("date_histogram",[ "field": "publish_date","interval": "1M","time_zone": "Asia/Shanghai","min_doc_count": 1])
detail.put("date_histogram",[ "field": "publish_date","interval": "1M","time_zone": "Asia/Shanghai","min_doc_count": 0])
detail.put("aggs",[ "1": ["moving_avg": ["buckets_path": "1-metric"]],"1-metric": ["avg": ["field": "avg_price"]]])
def must = []
@@ -143,15 +144,18 @@ class EsHousePriceStat extends BaseBatch{
if(!priceValue){
priceValue = row['1-metric']
}
if(priceValue){
if(priceValue && priceValue.value){
price = priceValue.value.toInteger()
}
result.price = price;
def date = row.key_as_string.replace("-","").substring(0, 6)
def curMonth = DateUtils.formatDateByPattern(new Date(),"yyyyMM")
if(date!=curMonth){
result.month = date.toInteger()
rets << result
if(date.toInteger()>200001 && result.price){
result.month = date.toInteger()
rets << result
}
}
}
}
@@ -164,6 +164,7 @@ class OpenApiUtils {
if(StringUtils.isEmpty(city) || StringUtils.isEmpty(address)){
return null;
}
address = address.trim().replace(" ","")
String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&types=&city=" + city + "&output=json&offset=20&page=1&key=2af0b8533415a612b98d59bb93d664b9&extensions=all"
List<com.alibaba.fastjson.JSONObject> poiList = null;
@@ -18,44 +18,62 @@ class CcbCityPriceStat {
task.run()
}
def run(){
def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
def citys = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","临汾","聊城","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","齐齐哈尔","眉山","郴州","营口","蚌埠","赤峰","大庆","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","朔州","随州","大理","通化","楚雄","巴彦淖尔","乌兰察布","松原","景德镇","新余","辽阳","阜新","黔南","酒泉","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","钦州","内江","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","鹤岗","萍乡","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","湘西","昭通","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"]
def rets = []
def startDate = DateUtils.parseFromTightByPattern("2022-02-01","yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2022-06-01","yyyy-MM-dd")
for(def city : citys){
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias"])
def priceMap = CommonDatautils.queryDistrictPrice(city, DateUtils.parseFromTightYYYYMMDD("2021-05-30"))
def cityRets = []
for(def dist : districts){
def tradeAvgPrice = findTrades(dist, "2021-02-01", "2021-06-01")
def weekendPrice = priceMap.get(city +"##"+dist.region+"##"+dist.name)
if(tradeAvgPrice && weekendPrice){
def row = [city, dist.region, dist.name, weekendPrice, tradeAvgPrice.toInteger()]
cityRets << row
println row.toString()
def matchTotalCount = findMatchTradeCount(city, startDate, endDate)
println city +" "+matchTotalCount
if(matchTotalCount){
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias"])
def priceMap = CommonDatautils.queryDistrictPrice(city, DateUtils.parseFromTightYYYYMMDD("2022-05-29"))
def cityRets = []
for(def dist : districts){
def tradeAvgPrice = findTrades(dist, "2022-02-01", "2022-06-01")
def weekendPrice = priceMap.get(city +"##"+dist.region+"##"+dist.name)
if(tradeAvgPrice && weekendPrice){
def row = [city, dist.region, dist.name, weekendPrice, tradeAvgPrice.toInteger()]
cityRets << row
println row.toString()
}
}
print city +" "+ cityRets.size()
rets.addAll(cityRets)
}
print city +" "+ cityRets.size()
rets.addAll(cityRets)
}
println "×ÜÁ¿£º"+rets.size()
ExcelUtils.write("d:\\数据\\92个城市价格数据.xlsx", rets, "所有")
ExcelUtils.write("d:\\数据\\234个城市价格数据.xlsx", rets, "所有")
}
def findMatchTradeCount(city, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
q.put("match_id", new BasicDBObject('$exists', true))
def count = DEAL_PRICE_COLL.count(q)
return count
}
def findTrades(dist, startDate, endDate){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
q.put("district_name", new BasicDBObject('$in', dist.alias))
}else{
q.put("district_name", dist.name)
}
// q.put("city", dist.city)
// q.put("region", dist.region)
//
// if(dist.alias){
// if(!dist.alias.contains(dist.name)){
// dist.alias << dist.name
// }
// q.put("district_name", new BasicDBObject('$in', dist.alias))
// }else{
// q.put("district_name", dist.name)
// }
q.put("match_id", dist._id.toString())
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def avg_price = 0
@@ -199,18 +199,18 @@ class CcbCodeUtils {
static def getBuildStructureCode(def buildStructure){
if(buildStructure){
if(buildStructure.contains("钢结构")){
return "001";
return "3";
}else if(buildStructure.contains("钢、钢筋混凝土结构")){
return "002";
return "1";
}else if(buildStructure.contains("钢筋混凝土结构")){
return "003";
return "1";
}else if(buildStructure.contains("砖木结构")){
return "004";
return "10";
}else if(buildStructure.contains("混合结构")){
return "005";
return "2";
}
}
return "006";
return "11";
}
static def getDistHouseTypeCode(def estateType){
@@ -50,4 +50,32 @@ class FileUtils {
}
out.close();
}
static def createTxtFile(ret, fileName, charset, isAppend){
File file = new File(fileName);
if (!file.getParentFile().exists()) {
boolean result = file.getParentFile().mkdirs();
if (!result) {
System.out.println("´´½¨Ä¿Â¼Ê§°Ü");
}
}
OutputStream out = new FileOutputStream(fileName, isAppend);
def sb = new StringBuffer();
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
def index = 0
for(def row : ret){
if(row){
index++
sb.append(row)
sb.append("\n");
}
}
if(sb.length()>0){
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
}
out.close();
}
}
@@ -0,0 +1,257 @@
package com.fangjia.script.gj.crawl
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.thirdPriceCrawl.Utils
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import net.sf.json.JSONObject
import org.jsoup.Jsoup
import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver
class AnjukeAppCrawl {
public static final SHORT_DATE = Utils.getLastMonth();
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
public static def SOURCE = "安居客";
DBCollection detail = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "app_anjuke_detail")
static main(args) {
def task = new AnjukeAppCrawl();
// task.run();
// task.run2()
task.addThirdPrice()
}
def run2(){
def url = "https://m.anjuke.com/sh/community/"
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
def links = getRegion(url, driver)
for(def link : links){
def doc = getDocument(url, driver)
def str = doc.toString().substring(doc.toString().indexOf("<script> window.__NUXT__"), doc.toString().indexOf("</script>"))
if(str){
str = str.substring(str.indexOf("} ("), str.indexOf("));"))
def arrs = str.split(",")
pritln arrs.size()
}
}
}
def getDocument(url, driver){
def stop = false
def error = 0
def doc = null
while(!stop){
try{
driver.get(url)
doc = Jsoup.parse(driver.getPageSource());
if(doc && !doc.toString().contains("访问验证-安居客") && !doc.toString().contains("502 Bad Gateway") && !doc.toString().contains("<title>登录</title>")){
stop = true;
}else{
error++
driver.manage().deleteAllCookies()
println "[${error}] error: "+url
Thread.sleep(5000)
}
}catch(Exception e){
error++
}
if(error>=20){
stop = true;
}
}
return doc
}
def getRegion(url, driver){
//def doc = getDocument(url, driver)
def header = [:]
header.put("accept","application/json, text/plain, */*")
header.put("accept-encoding","gzip, deflate, br")
header.put("accept-language","zh-CN,zh;q=0.9")
header.put("cookie","aQQ_ajkguid=E9796514-38C0-44D2-8CA3-1C58C3928B29; sessid=765EEB9D-E7A2-4B2C-8151-CA9CD939AB02; id58=CpQB22J8sqMA9stuJfeAAg==; twe=2; 58tj_uuid=bf6eaff8-851c-471a-a914-853242878ddc; als=0; seo_source_type=1; _ga=GA1.2.215255179.1652344695; isp=true; ajk-appVersion=; obtain_by=2; fzq_js_anjuke_ershoufang_m=c2c12b2d1b221734fa20ef752cd6bdc0_1652509223606_27; fzq_h=ef0186eeaba082ecd2a47ede7c7e3e64_1652533612190_a34c026e9c1842a3b35bdd37f3d2aa0b_2015773878; init_refer=https%253A%252F%252Fm.anjuke.com%252Fcq%252Fcommunity%252F; new_uv=8; wmda_uuid=b59ce2efbba070ecfa8fe93d79f30dd4; wmda_new_uuid=1; wmda_session_id_6145577459763=1652535525740-dae51249-3379-d4e7; wmda_visited_projects=%3B6145577459763; new_session=0; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; xxzl_deviceid=q9XQbdA9hhkHJTj+RgVpwc7jrBzVJp9J1uTlp4hDtRiXtA615K4HS4R0pJ1dgrBa; ctid=20; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; fzq_js_anjuke_xiaoqu_m=c80eeca4a3301dbf27b3c79b9f18fa3b_1652536277223_27; xzuid=81992155-5990-4a2f-8842-32bcbfbb1117")
header.put("referer","https://m.anjuke.com/fz/")
header.put("sec-fetch-dest","empty")
header.put("sec-fetch-mode","cors")
header.put("sec-fetch-site","same-origin")
header.put("sw8","1-ODA0ZDA3N2ZmMDA1NDgyYmExNTFlODhlZWIyNmFkMzk=-N2UwZDkxMGU2ZTdjNDdkZmJlZDBkMjUzMmNkODQ2NWU=-0-MTE3-aHR0cHM6Ly9tLmFuanVrZS5jb20=-L2VzZi1hamF4L2NvbW11bml0eS9saXN0-bS5hbmp1a2UuY29t-1")
header.put("sw8-correlation","c291cmNl:YmVpZG91")
header.put("user-agent","Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1")
def doc = ABuYunUtil.doGet(url, null, header, "utf-8")
def links = []
if(doc){
def eles = JQueryUtils.select(".column2 a", doc)
for(def e : eles){
def href = e.attr("href")
links << href
}
}
return links;
}
def run(){
//上海:11,北京:14
//"仓山":4071,
// def map = ["晋安":4070,"鼓楼":4068,"闽侯":4073,"台江":4069,"福清":16378,"长乐":15880,"马尾":4072,"连江":16380,"平潭":16387,"罗源":16382,"永泰":16392,"闽清":16384]
// def regionMap = ["浦东":7,"闵行":11, "松江":15,"宝山":13,"嘉定":14,"徐汇":5,"青浦":19,"静安":2,"普陀":10,"杨浦":9,"奉贤":17,"黄浦":4,"虹口":8,"长宁":6,"金山":18,"崇明":20]
def regionMap = [ "朝阳":642,"海淀":601,"昌平":843,"丰台":753,"大兴":865,"通州":818,"房山":802,"顺义":837,"西城":704,"东城":690,"密云":856,"石景山":787,"怀柔":858,"门头沟":815,"延庆":860,"平谷":863,"北京周边":2678]
def cityMap = ["北京":regionMap]
def cityId = 14
regionMap.each{region, area_id->
for(def i=1;i<40;i++){
def stop = false;
def all_stop = false
def error_count=0
while(!stop){//渝北:2359 南岸:2368 沙坪坝:2380
//城口
def flagMap = getInfo(stop, i, area_id, cityId, "上海")
stop = flagMap.stop
all_stop = flagMap.all_stop
error_count++
if(error_count>=20){
stop = true
all_stop = true
}
}
if(all_stop){
break;
}
}
}
}
def getInfo(def stop, def i, def area_id, def cityId, def city){
def map = [:]
def url = "https://m.anjuke.com/esf-ajax/community/list?city_id=${cityId}&page_size=100&page=${i}&area_id=${area_id}&is_tw=1"
def header = [:]
header.put("accept","application/json, text/plain, */*")
header.put("accept-encoding","gzip, deflate, br")
header.put("accept-language","zh-CN,zh;q=0.9")
header.put("cookie","aQQ_ajkguid=E9796514-38C0-44D2-8CA3-1C58C3928B29; sessid=765EEB9D-E7A2-4B2C-8151-CA9CD939AB02; id58=CpQB22J8sqMA9stuJfeAAg==; twe=2; 58tj_uuid=bf6eaff8-851c-471a-a914-853242878ddc; als=0; seo_source_type=1; _ga=GA1.2.215255179.1652344695; isp=true; ajk-appVersion=; obtain_by=2; fzq_js_anjuke_ershoufang_m=c2c12b2d1b221734fa20ef752cd6bdc0_1652509223606_27; fzq_h=ef0186eeaba082ecd2a47ede7c7e3e64_1652533612190_a34c026e9c1842a3b35bdd37f3d2aa0b_2015773878; init_refer=https%253A%252F%252Fm.anjuke.com%252Fcq%252Fcommunity%252F; new_uv=8; wmda_uuid=b59ce2efbba070ecfa8fe93d79f30dd4; wmda_new_uuid=1; wmda_session_id_6145577459763=1652535525740-dae51249-3379-d4e7; wmda_visited_projects=%3B6145577459763; new_session=0; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; xxzl_deviceid=q9XQbdA9hhkHJTj+RgVpwc7jrBzVJp9J1uTlp4hDtRiXtA615K4HS4R0pJ1dgrBa; ctid=20; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; fzq_js_anjuke_xiaoqu_m=c80eeca4a3301dbf27b3c79b9f18fa3b_1652536277223_27; xzuid=81992155-5990-4a2f-8842-32bcbfbb1117")
header.put("referer","https://m.anjuke.com/sh/community/")
header.put("sec-fetch-dest","empty")
header.put("sec-fetch-mode","cors")
header.put("sec-fetch-site","same-origin")
header.put("sw8","1-ODA0ZDA3N2ZmMDA1NDgyYmExNTFlODhlZWIyNmFkMzk=-N2UwZDkxMGU2ZTdjNDdkZmJlZDBkMjUzMmNkODQ2NWU=-0-MTE3-aHR0cHM6Ly9tLmFuanVrZS5jb20=-L2VzZi1hamF4L2NvbW11bml0eS9saXN0-bS5hbmp1a2UuY29t-1")
header.put("sw8-correlation","c291cmNl:YmVpZG91")
header.put("user-agent","Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1")
def ret = ABuYunUtil.doGet(url, null, null, "utf-8")
def all_stop = false;
def error_count = 0
if (ret) {
JSONObject jsonObj = JSONObject.fromObject(ret);
if(jsonObj && jsonObj.status=="ok" ){
def data = jsonObj.data
if(data && data.total.toInteger()>0){
def communities = data.communities
println "【${i}】 "+communities.size() +" "+url
if(communities.size()>0){
save(city, communities)
Thread.sleep(10000)
stop = true;
}else{
stop = true;
println url +" 返回0"
all_stop = true;
}
}else{
stop = false;
error_count ++
println url +" 无数据"
}
}else{
stop = false;
Thread.sleep(5000)
}
}
map.stop = stop
map.all_stop = all_stop;
return map;
}
def save(city, def ret){
def index = 0
for(def o : ret){
index++
def obj = findIsExist(o.jumpAction)
if(!obj){
def udate = new BasicDBObject()
o.each{k,v->
udate.put(k,v)
}
udate.put("city", city)
udate.put("c_date", new Date())
detail.save(udate)
println index+" [保存]"+o.jumpAction
}else{
println index+" [已存在]"+o.jumpAction
}
}
}
def findIsExist(def url){
def q = new BasicDBObject()
q.put("jumpAction",url)
return detail.findOne(q);
}
def addThirdPrice(){
def q = new BasicDBObject()
def dbcursor = detail.find(q)
def index = 0
for(def d : dbcursor){
index++
def city = d.base.cityName
def region = d.base.areaName
def name = d.base.name
def obj = findByParams(city, region, name);
if(!obj){
def data = new BasicDBObject()
data.put("district_id",d.base.id)
data.put("city", city)
data.put("region", region )
data.put("name", name)
data.put("address",d.base.address)
data.put("avg_price", d.priceInfo.price)
data.put("source", SOURCE)
data.put("s_date", SHORT_DATE)
data.put("c_date", d.c_date)
thirdPartyColl.save(data)
println index +" "+data
}
}
}
def findByParams(city, region, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("source", SOURCE)
q.put("s_date", SHORT_DATE)
return thirdPartyColl.findOne(q);
}
}
@@ -11,9 +11,15 @@ import org.openqa.selenium.By
import org.openqa.selenium.WebDriver
import org.openqa.selenium.WebElement
import org.openqa.selenium.chrome.ChromeDriver
import org.openqa.selenium.remote.DesiredCapabilities
import org.openqa.selenium.remote.RemoteWebDriver
class AnjukeDetailCrawl {
DBCollection detailColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_detail")
DBCollection urlColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "detail_url")
static main(args) {
def task = new AnjukeDetailCrawl();
task.run();
@@ -21,9 +27,15 @@ class AnjukeDetailCrawl {
def run(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
def url = "https://chongqing.anjuke.com/community/?from=navigation"
def url = "https://chongqing.anjuke.com/"
WebDriver driver = new ChromeDriver()
// DesiredCapabilities capabilities = DesiredCapabilities.chrome();
// URL urlInstance = new URL("http://192.168.0.100:30025"); //指定URL
// WebDriver driver = new RemoteWebDriver(urlInstance, capabilities);
driver.manage().deleteAllCookies()
def regions = getRegions(driver, url)
println "区域链接: "+regions.size()
getAllRegions(driver, regions)
}
@@ -31,44 +43,88 @@ class AnjukeDetailCrawl {
def getAllRegions(driver, regions){
for(def url : regions){
driver.get(url)
def doc = Jsoup.parse(driver.getPageSource());
def doc = getDocument(url, driver)
def total = getTotal(doc)
if(total) {
if (total > 1250) {
def links = getMoreLinks(url, driver)
println url +" "+links.size()
saveDetailLinks(links)
// parse(links,driver)
} else {
def links = getAllLinks(url, driver)
parse(links,driver)
println url +" "+links.size()
saveDetailLinks(links)
// parse(links,driver)
}
}
}
}
def saveDetailLinks(links){
for(def url : links){
def data = new BasicDBObject()
data.put("city","重庆")
data.put("url", url)
data.put("source", "安居客")
data.put("c_date", new Date())
urlColl.save(data)
}
}
def getMoreLinks(url, driver){
def doc = Jsoup.parse(driver.getPageSource());
def pricelinks = JQueryUtils.select(".filter-wrap:contains(均价) .line-item a", doc)
def links = []
if(pricelinks){
for(def link : pricelinks){
if(link.text()!="不限"){
def urlLink = link.attr("href")
getDocument(urlLink, driver)
// driver.get(urlLink)
links.addAll(getAllLinks(urlLink, driver))
println urlLink +" "+links.size()
}
}
}
return links;
}
def getAllLinks(url, driver){
def stop = false
def urls = []
while(!stop) {
def doc = Jsoup.parse(driver.getPageSource());
def doc = Jsoup.parse(driver.getPageSource());
def total = getTotal(doc)
def page = getPage(total, 25)
def pages = []
for(def i=1;i<=page;i++){
pages << url.substring(0, url.size()-1)+"-p"+i+"/"
}
for(def pageUrl : pages){
doc = getDocument(pageUrl, driver)
def details = JQueryUtils.select(".list-cell a.li-row", doc)
for (def d : details) {
urls << d.attr("href")
}
def nextEle = JQueryUtils.select(".next", doc)
if (nextEle) {
driver.findElement(By.cssSelector(".next")).click()
}else{
stop = true
}
if(urls.size()>20){
stop = true
def link = d.attr("href")
if(!urls.contains(link)){
urls << d.attr("href")
}
}
}
return urls
}
def getPage(def total, def pageSize){
if(total && total.toInteger()>0){
def page = total.toInteger()/pageSize;
if(page>(int)page){
page = (int)page + 1
return page
}
return (int)page;
}
return null
}
def getTotal(doc){
def totalText = JQueryUtils.selectText(".total-info", doc)
@@ -82,9 +138,19 @@ class AnjukeDetailCrawl {
}
def getRegions(driver, url){
// def doc = getDocument(url, driver)
driver.get(url)
Thread.sleep(3000)
def doc = Jsoup.parse(driver.getPageSource());
def els = JQueryUtils.select("a:contains(小区)", doc)
if(els){
def href = els[0].attr("href")
doc = Jsoup.parse(driver.getPageSource());
// driver.findElement(By.xpath("//a[contains(text(), '小区')]")).click()
// driver.findElement(By.linkText("小区")).click()
doc = getDocument(href, driver)
}
// def loginIfm = JQueryUtils.select("#iframeLoginIfm", doc)
// if(loginIfm){
// def href = loginIfm[0].attr("src")
@@ -107,23 +173,35 @@ class AnjukeDetailCrawl {
def regionEles = JQueryUtils.select(".region-parents li a", doc)
def urls = []
for(def e : regionEles){
urls << e.attr("href")
if(e.text()!="不限"){
urls << e.attr("href")
}
}
return urls;
}
def getNontCheckPage(url, driver, doc){
def stop = false;
while(!stop){
if(doc){
}
}
return doc;
}
def parse(def links, driver){
def rets = []
for(def link : links){
def district = parsing(link, driver)
if(district){
rets << district
// rets << district
if (rets.size() % 100 == 0) {
def add= saveData(rets)
println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
rets = []
}
// if (rets.size() % 100 == 0) {
saveObj(district)
println district.name+" "+district.url + " "+DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
// rets = []
// }
}
}
if (rets.size()> 0) {
@@ -136,33 +214,61 @@ class AnjukeDetailCrawl {
def saveData(def results){
def add = 0
for(def ret : results){
def q = new BasicDBObject()
q.put("url", ret.url)
def dbo = detailColl.findOne(q);
DBObject obj = new BasicDBObject();
ret.each{k,v->
obj.put(k,v)
}
if(dbo){
obj.m_date = new Date()
detailColl.update(q, new BasicDBObject('$set', obj))
}else{
add++
obj.c_date = new Date()
obj.m_date = new Date()
try{
detailColl.save(obj)
}catch(Exception e){
println obj.url
}
}
saveObj(ret)
}
return add
}
def saveObj(def ret){
def q = new BasicDBObject()
q.put("url", ret.url)
def dbo = detailColl.findOne(q);
DBObject obj = new BasicDBObject();
ret.each{k,v->
obj.put(k,v)
}
if(dbo){
obj.m_date = new Date()
detailColl.update(q, new BasicDBObject('$set', obj))
}else{
obj.c_date = new Date()
obj.m_date = new Date()
try{
detailColl.save(obj)
}catch(Exception e){
println obj.url
}
}
}
def getDocument(url, driver){
def stop = false
def error = 0
def doc = null
while(!stop){
try{
driver.get(url)
doc = Jsoup.parse(driver.getPageSource());
if(doc && !doc.toString().contains("访问验证-安居客") && !doc.toString().contains("502 Bad Gateway") && !doc.toString().contains("<title>登录</title>")){
stop = true;
}else{
error++
driver.manage().deleteAllCookies()
println "[${error}] error: "+url
Thread.sleep(5000)
}
}catch(Exception e){
error++
}
if(error>=20){
stop = true;
}
}
return doc
}
def parsing(url, driver){
@@ -191,10 +297,9 @@ class AnjukeDetailCrawl {
def infoDiv = JQueryUtils.select(".info-list", doc)
district.estate_type = getValue(infoDiv.select(".column-2:contains(物业类型) .value").text())
district.estate_charge = getValue(infoDiv.select(".column-2:contains(物业费) .value").text())
district.estate_company = getValue(infoDiv.select(".column-1:contains(物业公司) .value").text())
district.developer = getValue(infoDiv.select(".column-1:contains(开发商) .value").text())
district.quanshu_type = getValue(infoDiv.select(".column-2:contains(权属类别) .value").text())
district.finish_date = getValue(infoDiv.select(".column-2:contains(竣工时间) .value").text())
district.property_life = getValue(infoDiv.select(".column-2:contains(产权年限) .value").text())
def houseCountStr = infoDiv.select(".column-2:contains(总户数) .value").text()
if (houseCountStr) {
houseCountStr = houseCountStr.replace("户", "")
@@ -202,9 +307,13 @@ class AnjukeDetailCrawl {
district.house_count = houseCountStr.toInteger()
}
}
district.finish_date = getValue(infoDiv.select(".column-2:contains(竣工时间) .value").text())
district.build_area = getValue(infoDiv.select(".column-2:contains(总建面积) .value").text())
district.volumetric_rate = getValue(infoDiv.select(".column-2:contains(容积率) .value").text())
district.greening_rate = getValue(infoDiv.select(".column-2:contains(绿化率) .value").text())
district.build_type = getValue(infoDiv.select(".column-2:contains(建筑类型) .value").text())
district.business = getValue(infoDiv.select(".column-2:contains(所属商圈) .value").text())
def carSiteStr = getValue(infoDiv.select(".column-2:contains(停车位) .value").text())
if(carSiteStr){
if(carSiteStr.contains("(")){
@@ -214,16 +323,19 @@ class AnjukeDetailCrawl {
district.car_site = carSiteStr
}
}
district.volumetric_rate = getValue(infoDiv.select(".column-2:contains(容积率) .value").text())
district.greening_rate = getValue(infoDiv.select(".column-2:contains(绿化率:) .value").text())
district.build_type = getValue(infoDiv.select(".column-2:contains(建筑类型) .value").text())
district.property_life = getValue(infoDiv.select(".column-2:contains(产权年限) .value").text())
district.car_site_charge = getValue(infoDiv.select(".column-1:contains(停车费) .value").text())
district.car_site_str = infoDiv.select(".column-2:contains(停车位) .value").text()
district.estate_charge = getValue(infoDiv.select(".column-2:contains(物业费) .value").text())
district.stop_charge = getValue(infoDiv.select(".column-1:contains(停车费) .value").text())
district.car_site_charge = getValue(infoDiv.select(".column-1:contains(车位管理费) .value").text())
district.estate_company = getValue(infoDiv.select(".column-1:contains(物业公司) .value").text())
district.developer = getValue(infoDiv.select(".column-1:contains(开发商) .value").text())
district.url = url
district.c_date = new Date()
district.m_date = new Date()
district.crawl_type = "chrome"
district.root = "anjuke.com"
stop = true;
}else{
Thread.sleep(3000)
@@ -0,0 +1,89 @@
package com.fangjia.script.gj.crawl
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
class AnjukeDistrictMatch {
static main(args) {
def task = new AnjukeDistrictMatch();
task.run();
// task.run1()
}
def run1(){
def regionPointsMap = findRegionRange("重庆")
def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region))
}
def run(){
def filePath = "D:\\数据\\重庆农商行\\安居客\\重庆小区.xlsx"
ExcelUtils.multiThreadingModify(filePath,{row->
if(row['A'] && !row['J']){
def city = row['B']
def region = row['C']
def name = row['D']
def address = row['E']
// def flag = "别名匹配"
// def dist = matchDist(city, null, name)
// if(!row['J'] && name.contains("(")){
// name = name.replace("(${region})","").replace("(","").replace(")","")
// flag = "去括号匹配"
// dist = matchDist(city, region, name)
// }
// if(!dist){
// dist = CommonDatautils.findSeaweedByMappingAlias(city, region, name,"district")
// flag = "映射别名匹配"
// }
// if(!dist) {
// def rets = CommonDatautils.findDistrictByAlias(city, region, name, "district")
// if(rets && rets.size()==1){
// dist = rets[0]
// }
// }
//
// if(!dist){
// def dists = CommonDatautils.findDistrictByAddress(city, region, address,"district")
// flag = "地址匹配"
// if(dists.size()>1){
// println address +" 地址相同:"+dists.size()
// }else{
// if(dists.size()==1){
// dist = dists[0]
// }
// }
// }
// if(dist){
// row['J'] = flag
// row['K'] = dist._id.toString()
// row['L'] = dist.region
// row['M'] = dist.name
// row['N'] = dist.address
// }
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false,region,name)
if(jsonObject != null && jsonObject.precise!=-1) {
// row['J'] = "匹配接口"
// row['K'] = jsonObject.get("id")
// row['L'] = jsonObject.get("district")
// row['M'] = jsonObject.get("name")
// row['N'] = jsonObject.get("address")
row['M'] = jsonObject.get("name")
}
}
})
}
def matchDist(city, region, name){
def dist = CommonDatautils.findDistrict(city, region, name, "district")
return dist;
}
}
@@ -0,0 +1,189 @@
package com.fangjia.script.gj.crawl
import com.alibaba.fastjson.JSON
import com.alibaba.fastjson.JSONObject
import com.fangjia.core.RegionData
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.gj.EsUtils
import com.fangjia.script.temp.shenjian.util.MessageUtils
import com.fangjia.script.util.CommonUtils
import org.jsoup.Jsoup
import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver
class AnjukeHouseCrawl {
def targetUrl = "http://192.168.0.206:9200/amap_poi/_doc"
static main(args) {
def task = new AnjukeHouseCrawl();
task.run();
}
def run(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
for(def i=24;i<50;i++){
def url = "https://chengde.anjuke.com/sale/shuangqiao/p${i}/?from=navigation"
def doc = getDocument(url, driver)
def eles = JQueryUtils.select(".list-main .list-left .list .property a", doc)
println "页数:"+i+" 记录数:"+eles.size()
if(eles){
parser(eles, driver)
}
}
}
def parser(eles, driver){
def index = 0
for(def e : eles){
index ++
def href = e.attr("href")
def doc = getDocument(href, driver)
// parserHtml(doc)
def newUrl = href.substring(0, href.indexOf("?"))
sendMq(newUrl, doc)
println "[ "+index +"] "+ newUrl
}
}
def sendMq(href, doc){
JSON json = new JSONObject()
json.put("project", "sell_anjuke")
json.put("url", href)
JSON result = new JSONObject()
result.put("content", doc.toString())
json.put("result", result)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse", packedMsg)
}
def parserHtml(doc) {
String city = null
def result = [:]
result.siteCode = "安居客"
if (doc) {
result.title = JQueryUtils.selectText([".banner-title h1"], doc)
def heads = JQueryUtils.select(".crumbs a", doc)
if (heads && heads.size() >= 5) {
result.city = heads[0].text().replace("房产网", "")
def orgRegion = heads[2].text().replace("二手房", "")
result.region = RegionData.formatRegion(result.city, orgRegion.trim())
result.block = heads[3].text().replace("二手房", "")
result.district_name = heads[4].text()
} else if (heads && heads.size() >= 3) {
result.city = heads[0].text().replace("房产网", "")
result.district_name = heads[2].text()
def eles = JQueryUtils.select("li:contains(所在位置:) a", doc)
if (eles && eles.size() > 0) {
def orgRegion = eles[0].text()
result.region = RegionData.formatRegion(result.city, orgRegion.trim())
}
}
if (!result.district_name) {
println " [未解析出(城市-区域-名称)信息]"
}
println result
def priceStr = JQueryUtils.selectText(".maininfo-price-num", doc)
if (priceStr) {
if (priceStr.isNumber()) {
result.total_price = priceStr.toDouble() * 10000
}
}
def roomInfoStr = JQueryUtils.selectText(".maininfo-model-item-1 .maininfo-model-strong", doc)
if (roomInfoStr) {
Integer room = DataCrawlUtils.parseRoom(roomInfoStr)
Integer hall = DataCrawlUtils.parseHall(roomInfoStr)
Integer toilet = DataCrawlUtils.parseToilet(roomInfoStr)
if (room) {
result.room = room
}
if (hall) {
result.hall = hall
}
if (toilet) {
result.toilet = toilet
}
}
def areaStr = JQueryUtils.selectText(".maininfo-model-item-2 .maininfo-model-strong", doc).replace("㎡", "")
if (areaStr) {
result.area = areaStr.toDouble()
}
def directionStr = JQueryUtils.selectText(".maininfo-model-item-3 i", doc)
result.direction = DataCrawlUtils.parseDirection(directionStr)
def buildDateStr = JQueryUtils.selectText(".maininfo-model-item-3 .maininfo-model-weak", doc)
if (buildDateStr && buildDateStr.contains("/")) {
def arrs = buildDateStr.split("\\/")
if (arrs && arrs.size() == 2) {
result.build_date = arrs[0]
result.estate_type = arrs[1]
}
}
result.floor_info = JQueryUtils.selectText(".maininfo-model-item-1 .maininfo-model-weak", doc)
if (result.floor_info && result.floor_info.contains("共")) {
def height = result.floor_info
if (result.floor_info.contains("(")) {
height = height.substring(result.floor_info.indexOf("(") + 1, result.floor_info.indexOf(")"))
}
height = height.replace("共", "").replace("层", "")
if (height) {
result.height = height.toInteger()
}
}
if (result.area && result.total_price) {
result.avg_price = (result.total_price / result.area).toInteger()
}
def fitmitStr = JQueryUtils.selectText(".maininfo-model-item-2 .maininfo-model-weak", doc)
result.fitment = DataCrawlUtils.parseFitment(fitmitStr)
def dateStr = JQueryUtils.selectText(["td:contains(发布时间) .houseInfo-main-item-name"], doc)
}
}
def getDocument(url, driver){
def stop = false
def error = 0
def doc = null
while(!stop){
try{
driver.get(url)
doc = Jsoup.parse(driver.getPageSource());
Thread.sleep(5000)
def els = JQueryUtils.select(".list-main .list-left .list", doc)
if(!els){
els = JQueryUtils.select(".props-main", doc)
}
if(els && els.size()>0){
stop = true;
}else{
error++
driver.manage().deleteAllCookies()
println "[${error}] error: "+url
Thread.sleep(5000)
}
}catch(Exception e){
error++
}
if(error>=20){
stop = true;
}
}
return doc
}
}
@@ -0,0 +1,631 @@
package com.fangjia.script.gj.crawl
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import net.sf.json.JSONObject
import org.bson.types.ObjectId
class AnjukeToDB {
DBCollection detail = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "app_anjuke_detail")
static main(args) {
def task = new AnjukeToDB();
task.run();
// task.addNoMatch()
// task.updateEstateType()
// task.gaodeRangeData()
}
def updateEstateType(){
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\安居客重庆小区-曹玉彦.xlsx",0,1)
for(def row : datas){
def anjukeId = row['A']
def obj = findAnjukeDistrictDetail(anjukeId)
if(obj) {
def type = getEstatType(obj.extend.type)
def commId = row['L']
if(!commId){
commId = row['G']
}
if(type){
if(type=="别墅"){
def udata = new BasicDBObject()
udata.put("estate_type2", type)
udata.put("m_date", new Date())
println commId
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(commId.trim())), new BasicDBObject('$set', udata))
}
}else{
// def udata = new BasicDBObject()
// udata.put("status", 1)
// udata.put("name_cat",11)
// udata.put("m_date", new Date())
// DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(commId)), new BasicDBObject('$set', udata))
}
}
}
}
def gaodeRangeData(){
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\徐鹤高德地图.xlsx",0,1)
def index = 0
for(def d : datas){
index++
println index
def personName = d['AB']
if(personName){
def commid = d['J']
if(commid){
def obj = CommonDatautils.findDistrict("重庆",d['C'],personName,"district")
if(obj){
updateDist(d, obj._id.toString())
}
}else{
saveGaode(d)
}
}else{
def commid = d['J']
updateDist(d, commid)
}
}
}
def saveGaode(d){
def oldName = d['E']
def distName = nameFileter(d['C'], oldName)
def udata = new BasicDBObject()
def alias = []
def aliasStr = d['F']
if(aliasStr){
def arrs = aliasStr.split("/")
for(def s : arrs){
alias << s
}
}
if(alias.size()>0){
udata.put("alias", alias)
}
def address = AddressUtil.formatAddress("重庆", d['G'])
if(address){
udata.put("address", address)
}
if(d['N']){
udata.put("virescence_percent", d['N'])
}
if(d['O']){
udata.put("estate_charge", d['O'])
}
if(d['Q']){
udata.put("cubage_percent", d['Q'])
}
if(d['R']){
udata.put("developer", d['R'])
}
if( d['S']){
udata.put("build_area", d['S'])
}
if(d['T']){
def t = d['T'].substring(0, 3)
udata.put("build_date", t)
}
if(d['V'] && d['V'].isNumber()){
udata.put("car_site_count", d['V'].toInteger())
}
if(d['Y']){
def build_type = getBuildType2(d['Y'])
if(build_type){
udata.put("build_type", build_type)
}
}
if(d['Z']){
udata.put("property_life", d['Z'])
}
def lat = null
def lng = null
if(d['H']){
lat = d['H']
lng = d['I']
// 高德转百度
def baidu = BaiDuApiUtils.geoconv(lng.toDouble(), lat.toDouble(), 3)
def lng2 = baidu.x
def lat2 = baidu.y
Long dLat2 = lat2 * 10000000000L
Long dLng2 = lng2 * 10000000000L
udata.put("lat2", dLat2)
udata.put("lng2", dLng2)
Long dlat = lat.toDouble() * 10000000000L
Long dlng = lng.toDouble() * 10000000000L
udata.put("lat", dlat)
udata.put("lng", dlng)
}
def type = "普通住宅"
if(distName.contains("别墅")) {
type = "别墅"
}else if(distName.contains("商住")){
type = "商住"
}
if(udata){
udata.put("name_cat",0)
udata.put("status",0)
udata.put("cat","district")
udata.put("c_date", new Date())
udata.put("m_date", new Date())
udata.put("city", "重庆")
udata.put("region",d['C'])
udata.put("name", distName)
udata.put("estate_type2",type)
def markMap = [:]
markMap.put("data_source", "机器入库")
udata.put("mark_map",markMap)
DbUtils.seaweed.save(udata);
def stop = false
while (!stop){
if(udata._id.toString()){
addPrice(udata)
stop = true;
}
}
}
}
def updateDist(d, commid){
if(commid){
def dist = CommonDatautils.queryDistrictById(commid)
if(dist){
def oldName = d['E']
def distName = nameFileter(d['C'], oldName)
def udata = new BasicDBObject()
if(!distName.contains("号")){
def alias = dist.alias
if(!alias){
alias = []
}
if(!alias.contains(distName) && dist.name!=distName){
alias << distName
}
def aliasStr = d['F']
if(aliasStr){
def arrs = aliasStr.split("/")
for(def s : arrs){
alias << s
}
}
if(alias.size()>0 && alias!=dist.alias){
udata.put("alias", alias)
}
}
if(dist.address && !dist.address.contains("号")){
def address = AddressUtil.formatAddress("重庆", d['G'])
if(address){
udata.put("address", address)
}
}
if(!dist.virescence_percent && d['N']){
udata.put("virescence_percent", d['N'])
}
if(!dist.estate_charge && d['O']){
udata.put("estate_charge", d['O'])
}
if(!dist.cubage_percent && d['Q']){
udata.put("cubage_percent", d['Q'])
}
if(!dist.developer && d['R']){
udata.put("developer", d['R'])
}
if(!dist.build_area && d['S']){
udata.put("build_area", d['S'])
}
if(!dist.build_date && d['T']){
if(d['T'].size()>=4){
def t = d['T'].substring(0, 4)
udata.put("build_date", t)
}
}
if(!dist.car_site_count && d['V'] && d['V'].isNumber()){
udata.put("car_site_count", d['V'].toInteger())
}
if(!dist.build_type && d['Y']){
def build_type = getBuildType2(d['Y'])
if(build_type){
udata.put("build_type", build_type)
}
}
if(!dist.property_life && d['Z']){
udata.put("property_life", d['Z'])
}
def lat = null
def lng = null
if(d['H']){
lat = d['H']
lng = d['I']
def slat = dist.lat/10000000000.0
def slng = dist.lng/10000000000.0
def distance = GeoUtils.calculateDistance(lng.toDouble(), lat.toDouble(), slng, slat);
if(distance>1000){
// 高德转百度
def baidu = BaiDuApiUtils.geoconv(lng.toDouble(), lat.toDouble(), 3)
def lng2 = baidu.x
def lat2 = baidu.y
Long dLat2 = lat2 * 10000000000L
Long dLng2 = lng2 * 10000000000L
udata.put("lat2", dLat2)
udata.put("lng2", dLng2)
Long dlat = lat.toDouble() * 10000000000L
Long dlng = lng.toDouble() * 10000000000L
udata.put("lat", dlat)
udata.put("lng", dlng)
}
}
if(udata){
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
println "[修改] ${dist.name}"+ udata
}
}
}
}
def addNoMatch(){
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\安居客未匹配.xlsx",0,1)
for(def d : datas){
add(d)
}
}
def run(){
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\非人工处理.xlsx",0,1)
for(def data : datas){
// if(data['K']=="镕杰"){
def flag = data['M']
if(flag && flag=="新增"){
// add(data)
}else{
update(data)
}
// }
}
}
def add(def row){
def anjukeId = row['A']
def obj = findAnjukeDistrictDetail(anjukeId)
if(obj){
def type = getEstatType(obj.extend.type)
if(type){
def city = obj.base.cityName
def region = obj.base.areaName
def lat2 = null
def lng2 = null
def lat = null
def lng = null
if(obj.base.lat){
lat2 = obj.base.lat
lng2 = obj.base.lng
JSONObject convertObj = GaoDeApiUtils.convert(lng2+","+lat2, "baidu")
if(convertObj.getInt("status") == 1) {
String lo = convertObj.get("locations")
lng = Double.parseDouble(lo.split(",")[0])
lat = Double.parseDouble(lo.split(",")[1])
}
}
def distName = nameFileter(obj.base.areaName, obj.base.name)
def buildTypeStr = getBuildType(obj.base.buildTypeStr)
def address = AddressUtil.formatAddress(obj.base.city, obj.base.address)
def dist = CommonDatautils.findDistrict(city, region, distName, "district")
if(!dist){
def data = new BasicDBObject();
data.put("name_cat",0)
data.put("status",0)
data.put("cat","district")
data.put("c_date", new Date())
data.put("m_date", new Date())
data.put("city", city)
data.put("region",region)
data.put("name", distName)
data.put("estate_type2",type)
if(address){
data.put("address", obj.base.address)
}
if(lat){
Long alat = lat*10000000000L
Long alng = lng*10000000000L
data.put("lat", alat)
data.put("lng", alng)
Long alat2 = lat2.toDouble()*10000000000L
Long alng2 = lng2.toDouble()*10000000000L
data.put("lat2",alat2)
data.put("lng2",alng2)
}
data.put("finish_building_date",obj.extend.completionTime)
if(obj.extend.totalHouseHoldNum && obj.extend.totalHouseHoldNum.isNumber()){
data.put("household_count",obj.extend.totalHouseHoldNum.toInteger())
}
data.put("build_area", obj.extend.totalArea)
data.put("virescence_percent", obj.extend.landscapingRatio)
data.put("build_type", buildTypeStr)
data.put("business_quarter",obj.base.tradingAreaName)
def markMap = [:]
markMap.put("data_source", "机器入库")
data.put("mark_map",markMap)
DbUtils.seaweed.save(data);
def stop = false
while (!stop){
if(data._id.toString()){
addPrice(data)
stop = true;
}
}
}
}else{
println "【物业类型问题】 ${obj._id.toString()} "+obj.extend.type
}
}else{
println anjukeId+" 没找到"
}
}
def update(def row){
def commId = row['L']
if(!commId){
commId = row['G']
}
def dist = CommonDatautils.queryDistrictById(commId)
def anjukeId = row['A']
def obj = findAnjukeDistrictDetail(anjukeId)
if(obj && dist){
def udata = new BasicDBObject()
def address = AddressUtil.formatAddress(obj.base.cityName, obj.base.address)
if(address){
if(!dist.address){
if(obj.base.address){
udata.put("address",obj.base.address)
}
}else{
if(dist.address.contains("号")){
if(!dist.address.contains(obj.base.address) && !obj.base.address.contains(dist.address)){
def addr_alias = dist.addr_alias
if(!addr_alias){
addr_alias = []
}
if(!addr_alias.contains(obj.base.address) && obj.base.address.contains("号")){
addr_alias << obj.base.address
udata.put("addr_alias", addr_alias)
}
}
}else{
udata.put("address",obj.base.address)
}
}
}
def distName = nameFileter(obj.base.areaName, obj.base.name)
if(!distName.contains("号")){
def alias = dist.alias
if(!alias){
alias = []
}
if(alias.contains(distName)){
alias << distName
}
if(alias.size()>0 && alias!=dist.alias){
udata.put("alias", alias)
}
}
if(!dist.build_type){
def buildTypeStr = getBuildType(obj.base.buildTypeStr)
if(buildTypeStr){
udata.put("build_type",buildTypeStr)
}
}
if(!dist.household_count){
if(obj.extend.totalHouseHoldNum && obj.extend.totalHouseHoldNum.isNumber() && obj.extend.totalHouseHoldNum.toInteger()>0){
udata.put("household_count",obj.extend.totalHouseHoldNum.toInteger())
}
}
if(!dist.finish_building_date){
if(obj.extend.completionTime){
udata.put("finish_building_date",obj.extend.completionTime)
}
}
if(!dist.virescence_percent){
if(obj.extend.landscapingRatio){
udata.put("virescence_percent", obj.extend.landscapingRatio)
}
}
if(!dist.business_quarter){
if(obj.base.tradingAreaName){
udata.put("business_quarter",obj.base.tradingAreaName)
}
}
def lat2 = null
def lng2 = null
if(dist.lat2){
lat2 = dist.lat2/10000000000.0
lng2 = dist.lng2/10000000000.0
}
if(dist.lat && !lat2){
// 高德转百度
def baidu = BaiDuApiUtils.geoconv(dist.lng/10000000000.0, dist.lat/10000000000.0, 3)
lng2 = baidu.x
lat2 = baidu.y
Long dLat2 = lat2 * 10000000000L
Long dLng2 = lng2 * 10000000000L
udata.put("lat2", dLat2)
udata.put("lng2", dLng2)
}
if(obj.base.lat) {
def alat2 = obj.base.lat.toDouble()
def alng2 = obj.base.lng.toDouble()
def distance = GeoUtils.calculateDistance(lng2, lat2, alng2, alat2);
if(distance>1000){
println "........${obj.base.name}..............${dist.name}..........距离相差${distance}......................"
def lng = null
def lat = null
JSONObject convertObj = GaoDeApiUtils.convert(obj.base.lng+","+obj.base.lat, "baidu")
if(convertObj.getInt("status") == 1) {
String lo = convertObj.get("locations")
lng = Double.parseDouble(lo.split(",")[0])
lat = Double.parseDouble(lo.split(",")[1])
}
Long blat2 = alat2 *10000000000L
Long blng2 = alng2 *10000000000L
Long gLat2 = lat *10000000000L
Long gLng2 = lng *10000000000L
udata.put("lat2", blat2)
udata.put("lng2", blng2)
udata.put("lat", gLat2)
udata.put("lng", gLng2)
}
}
if(udata){
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
println "[修改] ${dist.name}"+ udata
}
}
}
def findAnjukeDistrictDetail(anjukeId){
def q = new BasicDBObject()
try{
q.put("_id",new ObjectId(anjukeId))
return detail.findOne(q)
}catch(Exception e){
print "[查询出错]anjukeId:"+anjukeId
}
return null
}
def nameFileter(def region, def name){
name = name.replace("·","").replace(".","").replace("|","")
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
name = name.replace("(${region})","").replace("(","").replace(")","")
return name;
}
def getEstatType(type){
if(type){
if(type=="公寓住宅"){
type = "普通住宅"
}else if(type.contains("商住")){
type = "商住"
}else if(type.contains("别墅")){
type = "别墅"
}else{
type = null
}
}
return type
}
def getBuildType(def buildType){
def types = ["多层","小高层","高层","超高层","花园洋房","老洋房","四合院","独栋","双拼","联排","叠加","老式别墅洋房"]
def list = []
if(buildType){
def arrs = buildType.split("\\|")
for(def s : arrs){
if(types.contains(s)){
list << s
}
}
if(list){
return list.join(",")
}
}
return null
}
def getBuildType2(def buildType){
def types = ["多层","小高层","高层","超高层","花园洋房","老洋房","四合院","独栋","双拼","联排","叠加","老式别墅洋房"]
def list = []
if(buildType){
def arrs = buildType.split(",| ")
for(def s : arrs){
if(types.contains(s)){
list << s
}
}
if(list){
return list.join(",")
}
}
return null
}
def addPrice(dist){
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-04-01")
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-05-25")
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
for(def weekend : weekends){
CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0)
}
}
}
@@ -0,0 +1,250 @@
package com.fangjia.script.gj.crawl
import com.fangjia.script.util.DateUtils
import org.apache.commons.lang.StringUtils
import java.util.regex.Matcher
import java.util.regex.Pattern
class DataCrawlUtils {
static final Map<Pattern, Closure> DEAL_DATE_RESOLVERS = [:]
static final Map<Pattern, Closure> ROOM_RESOLVERS = [:]
static final Map<Pattern, Closure> HALL_RESOLVERS = [:]
static final Map<Pattern, Closure> TOILET_RESOLVERS = [:]
static final Map<Pattern, Closure> TOTAL_PRICE_RESOLVERS = [:]
static final Map<Pattern, Closure> UNIT_PRICE_RESOLVERS = [:]
static final Map<Pattern, Closure> AREA_RESOLVERS = [:]
static final Map<Pattern, Closure> HEIGHT_RESOLVERS = [:]
static final Map<Pattern, Closure> FLOOR_RESOLVERS = [:]
static final List<String> DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
static {
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}-\\d{2}"), { String str ->
return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}-\\d{2}-\\d{2}"), { String str ->
str = "20" + str
return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}/\\d{2}/\\d{2}"), { String str ->
str = "20" + str
return DateUtils.parseFromTightByPattern(str, "yyyy/MM/dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}\\.\\d{2}\\.\\d{2}"), { String str ->
return DateUtils.parseFromTightByPattern(str, "yyyy.MM.dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}"), { String str ->
return DateUtils.parseFromTightByPattern(str, "yyyy-MM")
})
ROOM_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两](室|房)"), { String str ->
str = str.replaceAll("室|房", '').trim()
return str.toInteger()
})
HALL_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]厅"), { String str ->
str = str.replace("厅", '').trim()
return str.toInteger()
})
TOILET_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]卫"), { String str ->
str = str.replace("卫", '').trim()
return str.toInteger()
})
TOTAL_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?万"), { String str ->
str = str.replace("万", '').trim()
return (str.toDouble() * 10000).toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/㎡"), { String str ->
str = str.replace("元/㎡", '').trim()
return str.toDouble().toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/m?"), { String str ->
str = str.replace("元/m?", '').trim()
return str.toDouble().toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/平"), { String str ->
str = str.replace("元/平", '').trim()
return str.toDouble().toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元"), { String str ->
str = str.replace("元", '').trim()
return str.toDouble().toInteger()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平米"), { String str ->
str = str.replace("平米", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平方米"), { String str ->
str = str.replace("平方米", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平"), { String str ->
str = str.replace("平", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?㎡"), { String str ->
str = str.replace("㎡", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?m?"), { String str ->
str = str.replace("m?", '').trim()
return str.toDouble()
})
HEIGHT_RESOLVERS.put(Pattern.compile("共\\d+层"), { String str ->
str = str.replace("共", '').replace('层', '')
return str.toInteger()
})
HEIGHT_RESOLVERS.put(Pattern.compile("总\\d+层"), { String str ->
str = str.replace("总", '').replace('层', '')
return str.toInteger()
})
HEIGHT_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str ->
def split = str.split("/")
def n1 = split[0].toInteger()
def n2 = split[1].toInteger()
if (n1 <= n2) {
return n2
}
return null
})
FLOOR_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str ->
def split = str.split("/")
def n1 = split[0].toInteger()
def n2 = split[1].toInteger()
if (n1 <= n2) {
return n1
}
return n1
})
FLOOR_RESOLVERS.put(Pattern.compile("第\\d+(层|楼)"), { String str ->
str = str.replace("第", "").replaceAll("层|楼", "")
return str.toInteger()
})
FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
}
/**
* 解析室数
* @param content
* @return
*/
static Integer parseRoom(String content) {
return parse(content, ROOM_RESOLVERS)
}
/**
* 解析厅数
* @param content
* @return
*/
static Integer parseHall(String content) {
return parse(content, HALL_RESOLVERS)
}
/**
* 解析厅数
* @param content
* @return
*/
static Integer parseToilet(String content) {
return parse(content, TOILET_RESOLVERS)
}
static Object parse(String content, Map<Pattern, Closure> resolvers) {
if (content != null) {
content = content.replaceAll("\\s", "")
}
for (def resolver : resolvers.entrySet()) {
String match = match(content, resolver.getKey())
if (StringUtils.isNotBlank(match)) {
match = match.replace("一","1").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("两","2")
return resolver.getValue()(match)
}
}
return null
}
static String match(String str, Pattern pattern) {
if (str == null) {
return null
}
Matcher matcher = pattern.matcher(str)
if (matcher.find()) {
return matcher.group()
}
return null
}
static String parseDirection(String directionDesc) {
if (directionDesc == null) {
return null
}
directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
StringBuffer sb = new StringBuffer()
for (int i = 0; i < directionDesc.length(); i++) {
String c = directionDesc.substring(i, i + 1)
if (DIRECTIONS.contains(c)) {
if (directionDesc.substring(i).startsWith(c + "#_#")) {
sb.append(c + " ")
} else {
sb.append(c)
}
}
}
for (String d : DIRECTIONS) {
if (sb.toString().startsWith(d)) {
return d
}
}
for (String d : DIRECTIONS) {
if (sb.toString().contains(d)) {
return d
}
}
return null
}
static String parseFitment(String fitmentDesc) {
if (fitmentDesc == null) {
return null
}
fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
if (fitmentDesc.equals("非毛坯")) {
return null
}
Iterator<Map.Entry<String, List<String>>> it = FITMENTS_TYPES_MAPS
.entrySet().iterator()
Map.Entry<String, List<String>> entry
while (it.hasNext()) {
entry = it.next()
if (fitmentDesc.contains(entry.getKey())) {
return entry.getKey()
}
for (String fitmentKey : entry.getValue()) {
if (fitmentDesc.contains(fitmentKey)) {
return entry.getKey()
}
}
}
return null
}
}
@@ -0,0 +1,347 @@
package com.fangjia.script.gj.crawl
import com.fangjia.script.util.DateUtils
import org.apache.commons.lang.StringUtils
import java.util.regex.Matcher
import java.util.regex.Pattern
/**
* 成交数据解析工具类
*/
class DealPriceUtils {
static final Map<Pattern, Closure> DEAL_DATE_RESOLVERS = [:]
static final Map<Pattern, Closure> ROOM_RESOLVERS = [:]
static final Map<Pattern, Closure> HALL_RESOLVERS = [:]
static final Map<Pattern, Closure> TOILET_RESOLVERS = [:]
static final Map<Pattern, Closure> TOTAL_PRICE_RESOLVERS = [:]
static final Map<Pattern, Closure> UNIT_PRICE_RESOLVERS = [:]
static final Map<Pattern, Closure> AREA_RESOLVERS = [:]
static final Map<Pattern, Closure> HEIGHT_RESOLVERS = [:]
static final Map<Pattern, Closure> FLOOR_RESOLVERS = [:]
static final List<String> DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
static {
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}-\\d{2}"), { String str ->
return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}-\\d{2}-\\d{2}"), { String str ->
str = "20" + str
return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}/\\d{2}/\\d{2}"), { String str ->
str = "20" + str
return DateUtils.parseFromTightByPattern(str, "yyyy/MM/dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}\\.\\d{2}\\.\\d{2}"), { String str ->
return DateUtils.parseFromTightByPattern(str, "yyyy.MM.dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}"), { String str ->
return DateUtils.parseFromTightByPattern(str, "yyyy-MM")
})
ROOM_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两](室|房)"), { String str ->
str = str.replaceAll("室|房", '').trim()
return str.toInteger()
})
HALL_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]厅"), { String str ->
str = str.replace("厅", '').trim()
return str.toInteger()
})
TOILET_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]卫"), { String str ->
str = str.replace("卫", '').trim()
return str.toInteger()
})
TOTAL_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?万"), { String str ->
str = str.replace("万", '').trim()
return (str.toDouble() * 10000).toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/㎡"), { String str ->
str = str.replace("元/㎡", '').trim()
return str.toDouble().toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/m²"), { String str ->
str = str.replace("元/m²", '').trim()
return str.toDouble().toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/平"), { String str ->
str = str.replace("元/平", '').trim()
return str.toDouble().toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元"), { String str ->
str = str.replace("元", '').trim()
return str.toDouble().toInteger()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平米"), { String str ->
str = str.replace("平米", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平方米"), { String str ->
str = str.replace("平方米", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平"), { String str ->
str = str.replace("平", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?㎡"), { String str ->
str = str.replace("㎡", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?m²"), { String str ->
str = str.replace("m²", '').trim()
return str.toDouble()
})
HEIGHT_RESOLVERS.put(Pattern.compile("共\\d+层"), { String str ->
str = str.replace("共", '').replace('层', '')
return str.toInteger()
})
HEIGHT_RESOLVERS.put(Pattern.compile("总\\d+层"), { String str ->
str = str.replace("总", '').replace('层', '')
return str.toInteger()
})
HEIGHT_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str ->
def split = str.split("/")
def n1 = split[0].toInteger()
def n2 = split[1].toInteger()
if (n1 <= n2) {
return n2
}
return null
})
FLOOR_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str ->
def split = str.split("/")
def n1 = split[0].toInteger()
def n2 = split[1].toInteger()
if (n1 <= n2) {
return n1
}
return n1
})
FLOOR_RESOLVERS.put(Pattern.compile("第\\d+(层|楼)"), { String str ->
str = str.replace("第", "").replaceAll("层|楼", "")
return str.toInteger()
})
FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
}
/**
* 解析成交日期
* @param content
* @return
*/
static Date parseDealDate(String content) {
return parse(content, DEAL_DATE_RESOLVERS)
}
/**
* 解析室数
* @param content
* @return
*/
static Integer parseRoom(String content) {
return parse(content, ROOM_RESOLVERS)
}
/**
* 解析厅数
* @param content
* @return
*/
static Integer parseHall(String content) {
return parse(content, HALL_RESOLVERS)
}
/**
* 解析厅数
* @param content
* @return
*/
static Integer parseToilet(String content) {
return parse(content, TOILET_RESOLVERS)
}
/**
* 解析总价
* @param content
* @return
*/
static Integer parseTotalPrice(String content) {
return parse(content, TOTAL_PRICE_RESOLVERS)
}
/**
* 解析单价
* @param content
* @return
*/
static Integer parseUnitPrice(String content) {
return parse(content, UNIT_PRICE_RESOLVERS)
}
/**
* 解析面积
* @param content
* @return
*/
static Double parseArea(String content) {
return parse(content, AREA_RESOLVERS)
}
/**
* 解析楼层
* @param content
* @return
*/
static Integer parseFloor(String content) {
return parse(content, FLOOR_RESOLVERS)
}
/**
* 解析楼高
* @param content
* @return
*/
static Integer parseHeight(String content) {
return parse(content, HEIGHT_RESOLVERS)
}
/**
* 格式化小区名
* @param name
* @return
*/
static String formatName(String name) {
name = name.replace("-商住", "").replace("-别墅", "")
//汇福家园平安里(迁)2015-2017
name = name.replaceAll("\\d\\d\\d\\d-\\d\\d\\d\\d", "")
//国贸天悦一期2017
name = name.replaceAll("\\d\\d\\d\\d\$", "")
if (name ==~ /.+(\(|()[\d一二三四五六七八九十东南西北|\w]+(期|区)(\)|))$/) {
name = name.replaceAll("\\(|(|\\)|)", '')
} else if (name ==~ /.+(\(|()别墅(\)|))$/) {
name = name.replaceAll("\\(|(|\\)|)", '')
} else if (name ==~ /.+(\(|().+(\)|))$/) {
name = name.replaceAll("(\\(|().+(\\)|))", '')
} else if (name.contains("·") || (name.contains("-") && !(name ==~ /.*\d+-\d+.*/))) {
println("---------------------------->" + name)
name = name.replaceAll("·|-", '')
println(name)
}
if (name ==~ /.{2,}路.{3,}$/ && !(name ==~ /.{2,}路.+号/) && !(name ==~ /.{2,}路.+弄/) && !(name ==~ /.*小区$/)) {
name = name.replaceAll("^.{2,}路", "")
}
return name
}
static Object parse(String content, Map<Pattern, Closure> resolvers) {
if (content != null) {
content = content.replaceAll("\\s", "")
}
for (def resolver : resolvers.entrySet()) {
String match = match(content, resolver.getKey())
if (StringUtils.isNotBlank(match)) {
match = match.replace("一","1").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("两","2")
return resolver.getValue()(match)
}
}
return null
}
/**
* 匹配符合条件的字符串
*
* @param str
* @param pattern
* @return
*/
static String match(String str, Pattern pattern) {
if (str == null) {
return null
}
Matcher matcher = pattern.matcher(str)
if (matcher.find()) {
return matcher.group()
}
return null
}
static String parseFitment(String fitmentDesc) {
if (fitmentDesc == null) {
return null
}
fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
if (fitmentDesc.equals("非毛坯")) {
return null
}
Iterator<Map.Entry<String, List<String>>> it = FITMENTS_TYPES_MAPS
.entrySet().iterator()
Map.Entry<String, List<String>> entry
while (it.hasNext()) {
entry = it.next()
if (fitmentDesc.contains(entry.getKey())) {
return entry.getKey()
}
for (String fitmentKey : entry.getValue()) {
if (fitmentDesc.contains(fitmentKey)) {
return entry.getKey()
}
}
}
return null
}
static String parseDirection(String directionDesc) {
if (directionDesc == null) {
return null
}
directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
StringBuffer sb = new StringBuffer()
for (int i = 0; i < directionDesc.length(); i++) {
String c = directionDesc.substring(i, i + 1)
if (DIRECTIONS.contains(c)) {
if (directionDesc.substring(i).startsWith(c + "#_#")) {
sb.append(c + " ")
} else {
sb.append(c)
}
}
}
for (String d : DIRECTIONS) {
if (sb.toString().startsWith(d)) {
return d
}
}
for (String d : DIRECTIONS) {
if (sb.toString().contains(d)) {
return d
}
}
return null
}
static String parsePrice(String priceStr) {
def pattern =Pattern.compile("\\d+(\\.\\d+)?万")
String match = match(priceStr, pattern)
}
static void main(String[] args) {
println parsePrice("顶秀欣园西苑2室1厅1卫492.3万")
}
}
@@ -0,0 +1,167 @@
package com.fangjia.script.gj.crawl
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.thirdPriceCrawl.Utils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.jsoup.Jsoup
import org.openqa.selenium.By
import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver
class ZhugeFangDistCrawl {
public static final SHORT_DATE = Utils.getLastMonth();
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
static main(args) {
def task = new ZhugeFangDistCrawl();
task.run();
}
def run(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
def url = "https://fz.esfxiaoqu.zhuge.com/fuqing/"
WebDriver driver = new ChromeDriver()
driver.manage().deleteAllCookies()
getRegions(driver, url)
}
def getRegions(driver, url) {
for(def i=1;i<=11;i++){
def newUrl = url+"page${i}/"
println newUrl
def doc = getDocument(newUrl, driver)
def total = parseList(doc)
// def next = JQueryUtils.select("a:contains(下一页)", doc)
// if(next){
// driver.findElement(By.xpath("//a[contains(text(), '下一页')]")).click()
// }
}
}
def parseList(doc){
def city= "福州"
def els = JQueryUtils.select("#listTableBox li", doc)
if (els) {
for(def e : els){
def name = e.select(".house-name").text()
def str = e.select(".house-adr").text()
def address = null
if(str){
if(str.contains("]")){
address = str.substring(str.indexOf("]")+1).trim()
}else{
address = str
}
}
def regionStr = null
if(str && str.contains("]") && str.contains("[")){
regionStr = str.substring(str.indexOf("[")+1, str.indexOf("]"))
}
def region = null
def block = null
if(regionStr){
def arrs = regionStr.split("-")
if(arrs && arrs.size()>0){
if(arrs.size()==2){
block = arrs[1].trim()
}
if(arrs.size()>=1){
region = arrs[0].trim()
}
}
}
def links = e.select(".house-name a")
def href = null
if(links){
href = links[0].attr("href")
}
def priceStr = e.select(".price-number").text()
def price = null
if(priceStr && !priceStr.contains("暂无")){
if(priceStr.contains("元")){
priceStr = priceStr.substring(0, priceStr.indexOf("元")).trim()
}
if(priceStr && priceStr.isNumber()){
price = priceStr.toInteger()
}
}
def obj = findByParams(city, region, name)
if(obj){
if(price){
DBObject udata2 = new BasicDBObject()
udata2.put("avg_price", price)
udata2.put("m_date", new Date())
println "[upeate] ${region} "+name
thirdPartyColl.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata2));
}
}else{
def udata = new BasicDBObject()
udata.put("city", city)
udata.put("region", region)
udata.put("block", block)
udata.put("url", href)
udata.put("address", address)
udata.put("name", name)
udata.put("avg_price", price)
udata.put("s_date", SHORT_DATE)
udata.put("c_date", new Date())
udata.put("m_date", new Date())
udata.put("source", "诸葛找房")
thirdPartyColl.save(udata)
println "[save] ${region} "+name+" "+priceStr
}
}
}
return els.size()
}
def findByParams(city, region, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("source", "诸葛找房")
q.put("s_date", SHORT_DATE)
return thirdPartyColl.findOne(q);
}
def getDocument(url, driver){
def stop = false
def error = 0
def doc = null
while(!stop){
try{
driver.get(url)
doc = Jsoup.parse(driver.getPageSource());
Thread.sleep(10000)
def els = JQueryUtils.select("#listTableBox li", doc)
if(els && els.size()>0){
stop = true;
}else{
error++
driver.manage().deleteAllCookies()
println "[${error}] error: "+url
Thread.sleep(5000)
}
}catch(Exception e){
error++
}
if(error>=20){
stop = true;
}
}
return doc
}
}
@@ -172,7 +172,8 @@ class NameProcess {
if((name.endsWith("村") || name.endsWith("街"))&& name.size()<=3){
return null
}
if(name.endsWith("电梯")||name.endsWith("大道")||name.endsWith("厂") || name.endsWith("组") || (name.endsWith("阁") && name.length()<=3) || name.endsWith("条")){
//|| name.endsWith("组")
if(name.endsWith("电梯")||name.endsWith("大道")||name.endsWith("厂") || name.endsWith("条")){
return null
}
if(name.contains("、")||name.contains("|")||name.contains(".")){
@@ -71,7 +71,8 @@ class DistrictIndexBatch extends BaseBatch{
def run(){
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def citys = CommonDatautils.findAllCity();
def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"]
//def citys = CommonDatautils.findAllCity();
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2","household_count"])
def code = cityCodeDatas.get(city)
@@ -86,7 +87,7 @@ class DistrictIndexBatch extends BaseBatch{
city_avg_total_price = 531.2615913
}
if(city_avg_total_price){
// calcute(city, dists, code, city_avg_total_price)
calcute(city, dists, code, city_avg_total_price)
calcuteIndex(city, dists, code, city_avg_total_price)
}
@@ -0,0 +1,123 @@
package com.fangjia.script.gj.location
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils
class DistrictMapMatch {
static main(args) {
def task = new DistrictMapMatch()
task.mapMatch()
}
def mapMatch(){
def index = 0
ExcelUtils.modify("D:\\数据\\重庆农商行\\小区\\待审核小区(地图匹配).xlsx",{ d->
if(!d['H']){
index++
def rets = OpenApiUtils.getGaoDeSearchLocation(d['A'],d['B'],d['C'])
def map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets);
// if (!map || !map.lat) {
// rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']);
// map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets);
// }
if(map){
d['F'] = map.id
if(map.adname){
d['G'] = map.adname
}else if(map.region){
d['G'] = map.region
}
d['H'] = map.name
d['I'] = map.address
if(map.type){
d['J'] = map.type
}else if(map.estateType){
d['J'] = map.estateType
}
d['K'] = map.lng
d['L'] = map.lat
d['M'] = map.score
d['N'] = map.source
}
}
},0)
}
def findStandardObject2(def city, def region, def name, def address, def rets) {
if (rets) {
def bestObj = [:]
if (rets.size() >= 1) {
for (def mapObj : rets) {
def score = 0
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
score = 0.2
}
if(mapObj.name && name){
def newName = mapObj.name;
def names = []
if(mapObj.name && mapObj.name.indexOf("(")>-1){
def name1 = mapObj.name.substring(0, mapObj.name.indexOf("("))
names << name1
def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")"))
names << name2
}else{
names << newName
}
def maxNameScore = 0
for(def n : names){
double score1 = CosineSimilarAlgorithm.getSimilarity(n, name);
if(score1 && score1>0){
def nameScore = (score1 * 4)/10
if(nameScore > maxNameScore){
maxNameScore = nameScore
}
}
}
if(maxNameScore){
try{
maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore)
}catch(Exception e){
maxNameScore = 0
}
}
score = maxNameScore + score;
}
if(mapObj.address && address){
double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address);
if(addressScore && addressScore>0){
try{
addressScore = DataHandleUtils.getDoubleRound(2, addressScore)
}catch(Exception e){
addressScore = 0
}
score = addressScore*4/10 + score
}
}
if(!bestObj){
mapObj.score = score
bestObj = mapObj
}else{
if(score > bestObj.score){
mapObj.score = score
bestObj = mapObj
}
}
}
}
return bestObj
}
return null
}
}
@@ -21,10 +21,58 @@ class MatchTest {
def task = new MatchTest()
// task.run()
// task.addNum()
task.dongguang()
// task.dongguang()
// task.meituanDel()
task.addLevel()
}
def addLevel() {
def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"]//
def weekend = DateUtils.getFirstDayOfWeek(new Date());
def cityLevelMap = [:]
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region","name"])
def priceMap = CommonDatautils.queryDistrictPrice(city, weekend)
def datas = addPriceData(dists, priceMap);
datas = calcuteLevel(datas);
cityLevelMap.put(city, ["price":priceMap, "level":datas])
println city +" 楼盘评级同步完成"
}
def path = "D:\\数据\\美团\\10城小区明细-结果.xlsx"//美团金融小区测试list-所有数据-评级.xlsx
ExcelUtils.multiThreadingModify(path, { row ->
def name = row['D']
def address = row['E']
def city = RegionData.formatCity(row['A'])
if(name && !row['O']){
if(city){
def map = cityLevelMap.get(city)
def rMap = null
if(map){
rMap = getArroundDistrictPriceByAddress(city, address, map.level)
}
if(rMap){
if(rMap.rank){
row['Q'] = rMap.rank
}
// if(rMap.level){
// row['Q'] = rMap.level
// }
}
}
}
})
}
def meituanDel(){
def path = "D:\\数据\\美团\\美团需入库小区合并后.xlsx"
def index = 0
@@ -122,7 +122,7 @@ public class BeiKeCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
}else{
@@ -153,7 +153,7 @@ public class BeiKeCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd")
if(dealDate && dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e)
@@ -193,7 +193,7 @@ public class BeiKeCrawlByChrome {
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
}
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
detail.hall = roomStr.substring(roomStr.indexOf("ÊÒ") + 1, roomStr.indexOf("Ìü"))
detail.hall = roomStr.substring(roomStr.indexOf("ÊÒ") + 1, roomStr.indexOf("Ìü")).toInteger()
}
}
@@ -215,7 +215,8 @@ public class BeiKeCrawlByChrome {
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
}
detail.source = "贝壳找房"
def dbo = findTrade(detail)
def detailUrl = e.select(".title a").attr("href")
def dbo = findTrade(detailUrl)
if (!dbo) {
def houseInfo = e.select(".houseInfo").text()
def houseInfos = houseInfo.split("\\|")
@@ -228,7 +229,7 @@ public class BeiKeCrawlByChrome {
detail.height = floorStr.toInteger()
}
}
detail.url = e.select(".title a").attr("href")
detail.url = detailUrl
detail.create_date = new Date()
detail.m_date = new Date()
println detail
@@ -313,8 +314,8 @@ public class BeiKeCrawlByChrome {
def roomStr = JQueryUtils.selectText("li:contains(房屋户型)", doc).replace("房屋户型", "")
if (roomStr) {
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
detail.hall = roomStr.substring(roomStr.indexOf("ÊÒ") + 1, roomStr.indexOf("Ìü"))
detail.toilet = roomStr.substring(roomStr.indexOf("ÎÀ") - 1, roomStr.indexOf("ÎÀ"))
detail.hall = roomStr.substring(roomStr.indexOf("ÊÒ") + 1, roomStr.indexOf("Ìü")).toInteger()
detail.toilet = roomStr.substring(roomStr.indexOf("ÎÀ") - 1, roomStr.indexOf("ÎÀ")).toInteger()
}
detail.direction = JQueryUtils.selectText("li:contains(房屋朝向)", doc).replace("房屋朝向", "")
detail.fitment = JQueryUtils.selectText("li:contains(装修情况)", doc).replace("装修情况", "")
@@ -336,13 +337,14 @@ public class BeiKeCrawlByChrome {
}
def findTrade(def detail){
def findTrade(def detailUrl){
def q = new BasicDBObject()
q.put("city", detail.city)
q.put("region", detail.region)
q.put("district_name", detail.district_name)
q.put("area", detail.area)
q.put("total_price", detail.total_price)
// q.put("region", detail.region)
// q.put("district_name", detail.district_name)
// q.put("area", detail.area)
// q.put("total_price", detail.total_price)
q.put("url", detailUrl)
q.put("trade_date", detail.trade_date)
q.put("source", detail.source)
def dbo = DEAL_PRICE_COLL.findOne(q)
@@ -358,13 +360,13 @@ public class BeiKeCrawlByChrome {
if(cityEles){
for(def e : cityEles){
def href = e.attr("href")
if(e.text().contains("ÖØÇì")){
// if(e.text().contains("¸£ÖÝ")){
def cityMap = [:]
cityMap.city = e.text()
cityMap.url = "https:"+href
list << cityMap
//cityMap.put(e.text(), "https:"+href)
}
// }
}
}
return list
@@ -29,7 +29,7 @@ class JdPaimaiCrawl {
task.run();
}
def run(){
def existsIds = findAllIds()
def existsIds = []//findAllIds()
println "ÒÑ´æÔÚ:"+existsIds.size()
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
@@ -43,78 +43,80 @@ class JdPaimaiCrawl {
def getList(driver){
def list = []
for(def i=1;i<=32;i++){
if(i==8){
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=560&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=573&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=579&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=584&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=589&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=593&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=598&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=604&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=609&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=613&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=617&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=621&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=632&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=6858&_=1564047626268"
}else if(i==5){
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=142&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=148&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=164&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=199&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=224&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=239&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=248&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=258&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=264&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=274&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=275&_=1564047626268"
}else if(i==17){
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1432&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1441&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1458&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1466&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1475&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1477&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1479&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=3154&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1381&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1387&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1396&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1405&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1413&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1421&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2922&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2980&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2983&_=1564047626268"
}else if(i==19){
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1601&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1611&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1617&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1627&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1634&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1709&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1643&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1650&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1655&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1666&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1672&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1677&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1684&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1690&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1698&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1704&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1705&_=1564047626268"
}else{
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&_=1564047626268"
}
}
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId=4&_=1564047626268"
// for(def i=1;i<=32;i++){
// if(i==8){
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=560&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=573&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=579&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=584&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=589&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=593&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=598&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=604&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=609&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=613&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=617&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=621&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=632&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=6858&_=1564047626268"
// }else if(i==5){
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=142&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=148&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=164&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=199&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=224&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=239&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=248&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=258&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=264&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=274&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=275&_=1564047626268"
// }else if(i==17){
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1432&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1441&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1458&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1466&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1475&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1477&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1479&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=3154&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1381&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1387&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1396&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1405&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1413&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1421&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2922&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2980&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2983&_=1564047626268"
// }else if(i==19){
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1601&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1611&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1617&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1627&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1634&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1709&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1643&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1650&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1655&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1666&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1672&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1677&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1684&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1690&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1698&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1704&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1705&_=1564047626268"
// }else{
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&_=1564047626268"
// }
// }
def links = []
for(def url : list){
driver.get(url)
@@ -430,10 +432,10 @@ class JdPaimaiCrawl {
def s1 = JQueryUtils.selectText(["li:contains(¾ºÂò¹«¸æ) .floor-content"], doc)
if(s1) {
def start = s1.indexOf("·¨Ôº½«ÓÚ")
def start = s1.indexOf("将于")
def end = s1.indexOf("Ö¹")
if(start>-1 && end>-1 && end>start){
s1 = s1.substring(s1.indexOf("·¨Ôº½«ÓÚ"), s1.indexOf("Ö¹"))
s1 = s1.substring(s1.indexOf("将于"), s1.indexOf("止"))
}
def dateRangs = DataUtil.getDateRange(s1)
if(dateRangs && dateRangs.size()==2) {
@@ -476,6 +478,11 @@ class JdPaimaiCrawl {
}
}
if(!result.startDate){
def sTime = JQueryUtils.selectText(".startTime", doc).replace("开拍时间:","")
result.startDate = DateUtils.parseFromTightByPattern(sTime, "yyyy年MM月dd日")
}
if(result.finishDate){
result.finishDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.finishDate, "2000-01-01")
}
@@ -38,6 +38,26 @@ public class LianjiaCrawlByChrome {
// task.run1()
task.addDealRegion();
task.standardRegion()
// task.updadteDealPriceField()
}
def updadteDealPriceField(){
def citys = CommonDatautils.findAllCity()
def index = 0
for(def city : citys){
index ++
def q = new BasicDBObject()
q.put("city", city)
q.put("hall", "0")
def count = DEAL_PRICE_COLL.count(q)
println "[${index}] "+city +" "+count
if(count && count>0){
def udata = new BasicDBObject()
udata.put("hall", 0)
DEAL_PRICE_COLL.update(q, new BasicDBObject('$set', udata), false, true);
println "[ÐÞ¸Ä]"+city +" "+ count
}
}
}
def run1(){
@@ -187,7 +207,7 @@ public class LianjiaCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
}else{
@@ -218,7 +238,7 @@ public class LianjiaCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e)
@@ -258,7 +278,7 @@ public class LianjiaCrawlByChrome {
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
}
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
detail.hall = roomStr.substring(roomStr.indexOf("ÊÒ") + 1, roomStr.indexOf("Ìü"))
detail.hall = roomStr.substring(roomStr.indexOf("ÊÒ") + 1, roomStr.indexOf("Ìü")).toInteger()
}
}
@@ -378,8 +398,8 @@ public class LianjiaCrawlByChrome {
def roomStr = JQueryUtils.selectText("li:contains(房屋户型)", doc).replace("房屋户型", "")
if (roomStr) {
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
detail.hall = roomStr.substring(roomStr.indexOf("ÊÒ") + 1, roomStr.indexOf("Ìü"))
detail.toilet = roomStr.substring(roomStr.indexOf("ÎÀ") - 1, roomStr.indexOf("ÎÀ"))
detail.hall = roomStr.substring(roomStr.indexOf("ÊÒ") + 1, roomStr.indexOf("Ìü")).toInteger()
detail.toilet = roomStr.substring(roomStr.indexOf("ÎÀ") - 1, roomStr.indexOf("ÎÀ")).toInteger()
}
detail.direction = JQueryUtils.selectText("li:contains(房屋朝向)", doc).replace("房屋朝向", "")
detail.fitment = JQueryUtils.selectText("li:contains(装修情况)", doc).replace("装修情况", "")
@@ -420,11 +440,11 @@ public class LianjiaCrawlByChrome {
def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
def cityMap = [:]
def citys= ["³¤´º","¼ªÁÖ"]
def citys= ["¸£ÖÝ"]
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
if (!href.contains("fang.lianjia")){
cityMap.put(e.text(), href)
}
}
@@ -4,9 +4,12 @@ import com.alibaba.fastjson.JSON
import com.fangjia.core.util.DataUtil
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.rabbitmq.client.impl.AMQImpl
import org.jsoup.Jsoup
import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver
@@ -19,9 +22,102 @@ class PaimaiTest {
DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai")
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
static main(args) {
def task = new PaimaiTest();
task.run();
// task.run();
// task.stat()
task.updatePaimai()
// task.updateFieldPaimai()
}
def updateFieldPaimai(){
def q = new BasicDBObject()
q.put("city", new BasicDBObject('$in',["重庆","重庆市"]))
def index = 0
def dbcursor = paimai43_COLL.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
for(def obj : dbcursor) {
index++
println index
if(obj.address && obj.address.contains("可贷款")){
obj.put("address", obj.address.substring(0,obj.address.indexOf("可贷款")))
paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj))
}
}
}
def updatePaimai(){
def q = new BasicDBObject()
q.put("url", Pattern.compile(".*jd.com.*"))
def dbcursor = paimaiResultColl.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def index =0
for(def d : dbcursor){
index++
println index
if(!d.url.contains("jd.com")){
continue
}
if(!d.proince.contains("重庆")){
continue
}
println d.url
def query = new BasicDBObject()
query.put("url", d.url)
def dbo = paimai43_COLL.findOne(query);
DBObject obj = new BasicDBObject();
d.each{k,v->
obj.put(k,v)
}
if(dbo){
obj.m_date = new Date()
paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj))
}else{
obj.c_date = new Date()
obj.m_date = new Date()
try{
paimai43_COLL.save(obj)
}catch(Exception e){
println obj.url
}
}
}
}
def stat(){
def q = new BasicDBObject()
q.put("city", new BasicDBObject('$in', ["重庆","重庆市"]))
def dbcursor = paimaiResultColl.find(q)
def keys = []
for(def db : dbcursor){
if(db.matchCity){
def key = db.matchCity +"##"+db.matchRegion+"##"+db.matchName;
if(!keys.contains(key)){
keys << key
}
// try{
// def jsonObject = OpenApiClient.addressMatch("重庆", db.address, "property", false, null, null)
// if(jsonObject != null && jsonObject.precise!=-1) {
// db.matchCity = jsonObject.get("city")
// db.matchRegion = jsonObject.get("district")
// db.matchName = jsonObject.get("name")
// paimaiResultColl.update(new BasicDBObject("_id", db._id),new BasicDBObject('$set',db))
// println db.address+" "+db.matchName
// }
// }catch(Exception e){
// println "error "+db.address
// }
}
}
println keys.size()
}
def run(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
@@ -11,14 +11,17 @@ import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.rabbitmq.client.AMQP
import net.sf.json.JSONObject
import org.apache.poi.hssf.usermodel.HSSFWorkbook
import org.bson.types.ObjectId
import java.text.DecimalFormat
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher
import java.util.regex.Pattern
@@ -34,6 +37,9 @@ class CZDataBatch extends BaseBatch{
DBCollection DIST_COLL = MongoUtils.getCollectionByName("business", "jn_bank_district", "mongo.db.fangjia.com")
//final def czIdMap = readCZDistrict("d:\\常州数据结果\\原始数据\\常州江南银行住宅编号.xls");
//final def czShopMap = readCZShop("d:\\常州数据结果\\原始数据\\常州江南银行商铺--更新ing(8).xls");
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
static main(args) {
runBatch(new CZDataBatch(), args);
}
@@ -41,11 +47,17 @@ class CZDataBatch extends BaseBatch{
@Override
void process(Object args) {
def citys = CommonDatautils.findAllCity();
citys.remove("常州");
citys.add(0, "常州")
for(def city : citys){
def weekend = DateUtils.getFirstDayOfWeek(new Date())
def statMap = CommonDatautils.queryAllDistrictPriceByWeekend(city, weekend)
def commids = findAllJnBankDatas(city)
def districts = converList(CommonDatautils.findAllDistricts(city, "district", fields));
districtRun(districts);
districtRun(districts, statMap, commids);
def shops = CommonDatautils.findAllDistricts(city, "shop", fields);
shops = mergeShop(shops);
@@ -112,29 +124,48 @@ class CZDataBatch extends BaseBatch{
return rMap;
}
def districtRun(def districts){
def districtRun(def districts, def statMap, commids){
def total = districts.size();
def index = 0;
ExecutorService executorService = Executors.newFixedThreadPool(20)
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(total)
for(def i=0;i<total;i++){
final def district = districts[i]
executorService.submit(new Runnable() {
@Override
fixedThreadPool.execute(new Runnable() {
void run() {
index ++;
println index+"/"+total +" ["+district.city+"-"+district.region+"-"+district.name+"]";
converDistrict(district);
try{
totalCount.addAndGet(1)
converDistrict(district, statMap, commids);
println totalCount.get()+"/"+total +" ["+district.city+"-"+district.region+"-"+district.name+"]";
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
executorService.shutdown()
executorService.awaitTermination(30, TimeUnit.MINUTES)
latch.await()
// def index = 0;
// ExecutorService executorService = Executors.newFixedThreadPool(20)
// for(def i=0;i<total;i++){
//
// final def district = districts[i]
// executorService.submit(new Runnable() {
// @Override
// void run() {
// index ++;
// println index+"/"+total +" ["+district.city+"-"+district.region+"-"+district.name+"]";
// converDistrict(district);
// }
// });
// }
// executorService.shutdown()
// executorService.awaitTermination(30, TimeUnit.MINUTES)
}
def converDistrict(def dist){
def converDistrict(def dist, def statMap, def commids){
def map = [:]
//楼盘编号
/* def c_id = czIdMap.get(dist.city+"#"+dist.region+"#"+dist.name)
@@ -155,7 +186,11 @@ class CZDataBatch extends BaseBatch{
map.estate_type2 = dist.estate_type2
map.build_structure = dist.build_structure
// 楼盘均价
map.avg_price = getDistrictPrice(dist.city, dist.region, dist.name, dist.address, dist.block);
def priceObj = statMap.get(dist.region+"#"+dist.name)
if(priceObj && priceObj.avg_price){
map.avg_price = priceObj.avg_price
}
//map.avg_price = getDistrictPrice(dist.city, dist.region, dist.name, dist.address, dist.block);
// 省 市 区
if(dist.city == "常州"){
map.province_code = regionMap.get("江苏");
@@ -166,23 +201,25 @@ class CZDataBatch extends BaseBatch{
map.city = dist.city;
map.region = dist.region;
if(!commids.contains(map.num_id)){
// 公共服务配套
map.public_service_set_code = getPublicServiceSetCode(dist.city, dist.lat2, dist.lng2);
// 基础设施
map.facilities_code = "01";
// 居住社区成熟度
map.sophistication_code = getSophisticationCode(dist.city, dist.lat2, dist.lng2);
// 交通条件
map.traffic_code = getTrafficCode(dist.city, dist.lat2, dist.lng2);
// 泊车条件 /
map.stop_car_code = getStopCarCode(dist);
// /学区房类型
map.school_code = getSchoolCode(dist.city, dist.region, dist.lat2, dist.lng2) ;
}
// 地址
map.address = dist.address
// 所属区域
map.in_region_code = getRegionCode(dist.region);
// 公共服务配套
map.public_service_set_code = getPublicServiceSetCode(dist.city, dist.lat2, dist.lng2);
// 基础设施
map.facilities_code = "01";
// 居住社区成熟度
map.sophistication_code = getSophisticationCode(dist.city, dist.lat2, dist.lng2);
// 交通条件
map.traffic_code = getTrafficCode(dist.city, dist.lat2, dist.lng2);
// 泊车条件 /
map.stop_car_code = getStopCarCode(dist);
// /学区房类型
map.school_code = getSchoolCode(dist.city, dist.region, dist.lat2, dist.lng2) ;
//绿化率
map.virescence_percent = getVirescencePercent(dist.virescence_percent, map.build_date);
//容积率
@@ -921,20 +958,20 @@ class CZDataBatch extends BaseBatch{
if(price){
return price.avg_price;
}else{
//样例估价
def s = OpenApiClient.evaluateDistrictPriceBySimilarHouse3(city, region, name, 100, false);
try{
JSONObject jsonObject = JSONObject.fromObject(s);
def map = DataHandleUtils.JsonToMap(jsonObject);
if(map.code==200){
if(map.result){
return map.result.avgPrice;
}
}
}catch(Exception e){
}
// //样例估价
// def s = OpenApiClient.evaluateDistrictPriceBySimilarHouse3(city, region, name, 100, false);
// try{
// JSONObject jsonObject = JSONObject.fromObject(s);
// def map = DataHandleUtils.JsonToMap(jsonObject);
// if(map.code==200){
// if(map.result){
// return map.result.avgPrice;
// }
//
// }
// }catch(Exception e){
//
// }
//分期找总期
@@ -1078,4 +1115,18 @@ class CZDataBatch extends BaseBatch{
return list;
}
def findAllJnBankDatas(city){
def q = new BasicDBObject()
q.put("city", city)
def keys = new BasicDBObject()
keys.put("num_id", 1)
def dbcursor = DIST_COLL.find(q, keys)
def set = new HashSet()
for(def d : dbcursor){
set << d.num_id
}
return set;
}
}
@@ -1,5 +1,6 @@
package com.fangjia.script.loudong
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
@@ -16,8 +17,15 @@ class CqfdcChromeCrawl {
static main(args) {
def task = new CqfdcChromeCrawl();
task.run()
// def task = new CqfdcChromeCrawl();
// task.run()
def name = "E3뵀짜" //69눙(방뺍C-7뵀짜) 방뺍5-5뵀짜
def s = DataHandleUtils.findMatchPattern(name, "[\\w+]?-?\\w+[#|눙|떠|뵀]짜?.*?")
println s
def unitNumStr = DataHandleUtils.findMatchPattern(name, "\\d+데禱")
println unitNumStr
}
def run(){
@@ -1,10 +1,14 @@
package com.fangjia.script.loudong
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBObject
@@ -14,6 +18,8 @@ import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher
import java.util.regex.Pattern
class CqfdcMatch {
@@ -23,6 +29,10 @@ class CqfdcMatch {
static DBCollection loudongFdcConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong_cqfdc", "192.168.0.235", 27018)
static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong", "192.168.0.235", 27018)
public static final DBCollection LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia_base", "loudong", lineUrl)
public static final DBCollection HOUSE_LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", lineUrl)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
static main(args) {
@@ -32,7 +42,41 @@ class CqfdcMatch {
// task.export2()
// task.matchLoudong()
task.addLoudong()
// task.addLoudong()
// task.addFdcLoudong()
task.updateHouseLoudong()
}
def updateHouseLoudong(){
def q = new BasicDBObject()
q.put("city", "重庆")
def keys = new BasicDBObject()
keys.put("room_num", 1)
keys.put("house_floor", 1)
def dbcursor = HOUSE_LOUDONG_COLL.find(q).batchSize(5000)
def index = 0
for(def d : dbcursor){
index++
def room_num = d.room_num
def house_floor = d.house_floor
try{
if(room_num && house_floor && room_num.isNumber()){
if(house_floor.toInteger()>0){
if(!room_num.toString().startsWith(house_floor.toString()) && room_num.toInteger()<=10){
def newValue = house_floor +"-"+room_num
println "[${index}] "+room_num +" "+house_floor+" "+newValue
def udata = new BasicDBObject()
udata.put("room_num",newValue)
HOUSE_LOUDONG_COLL.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata), false, false);
}
}
}
}catch(Exception e){
println d._id.toString()
}
}
}
def export(){
def q = new BasicDBObject()
@@ -157,29 +201,378 @@ class CqfdcMatch {
return rets;
}
def addHouseLoudong(){
def q = new BasicDBObject()
def dbcursor = loudongConn.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def keys = []
def index = 0
for(def d : dbcursor) {
index++
println index
if(d.room_num && (d.room_num.contains("商") || d.room_num.contains("社区用房"))){
continue
}
if(d.area.toDouble()<=25){
continue
}
def newRegion = RegionData.formatRegion(d.city, d.region)
if (newRegion) {
def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?")
def house_num = null
if (houseNumStr) {
house_num = houseNumStr.replace("号楼", "").replace("幢", "").replace("栋", "").replace("号", "")
}
if (house_num) {
def unitNum = DataHandleUtils.findMatchPattern(d.house_num, "\\d+单元")
if (unitNum) {
unitNum = unitNum.replace("单元", "")
}
def key = d.city + "##" + newRegion + "##" + d.district_name + "##" + house_num+ "##" + unitNum +"##"+d.room_num
if(!keys.contains(key) && d.room_num) {
keys << key
def count = findHouseLoudongDatas(d.city, newRegion, d.district_name, house_num, unitNum, d.room_num)
if(!count){
def dist = null
if(d.match_city){
dist = CommonDatautils.findDistrict(d.match_city, d.match_region, d.match_name,"district")
}
def udata = new BasicDBObject()
udata.put("city", d.city)
udata.put("region", newRegion)
udata.put("name", d.district_name)
udata.put("house_num", house_num)
if(unitNum){
udata.put("house_num_unit", unitNum)
}
udata.put("room_num", d.room_num)
udata.put("house_area",d.area.toDouble())
udata.put("status", 0)
udata.put("from", "房谱网")
if(d.floor_heigth && d.floor_heigth.isNumber()){
udata.put("house_height", d.floor_heigth.toInteger())
}
if(d.floor && d.floor.isNumber()){
udata.put("house_floor", d.floor.toInteger())
}
if(dist){
udata.put("district_id", dist._id.toString())
udata.put("name_in_seaweed", dist.name)
}
if(d.address){
def address = AddressUtil.formatAddress(d.city, d.address)
if(address){
udata.put("address", address)
}
}
udata.put("c_date", new Date())
udata.put("m_date", new Date())
HOUSE_LOUDONG_LINE_COLL.save(udata)
}
}
}
}
}
}
def findHouseLoudongDatas(city, region, district_name, house_num, unitNum, room_num){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", district_name)
q.put("house_num", house_num)
if(unitNum){
q.put("unit_num", unitNum)
}
q.put("room_num", room_num)
def count = HOUSE_LOUDONG_LINE_COLL.count(q)
return count;
}
def findAllLoundongByHouseNum(){
def q = new BasicDBObject()
def dbcursor = loudongConn.find(q).batchSize(5000)
def rets = []
def map = [:]
def dbcursor = loudongConn.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(5000)
def keys = []
for(def d : dbcursor){
def key = d.city +"##"+d.region+"##"+d.district_name+"##"+d.house_num
if(d.unit_num){
key = key +"##"+d.unit_num
}
def detailMap= map.get(key)
if(!detailMap){
detailMap = [:]
map.put(key,detailMap)
}
def newRegion = RegionData.formatRegion(d.city, d.region)
if(newRegion){
def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?")
def house_num = null
if(houseNumStr){
house_num = houseNumStr.replace("号楼","").replace("幢","").replace("栋","").replace("号","")
}
if(house_num){
def unitNum = DataHandleUtils.findMatchPattern(d.house_num, "\\d+单元")
if(unitNum){
unitNum = unitNum.replace("单元","")
}
def key = d.city +"##"+newRegion+"##"+d.district_name+"##"+house_num
if(unitNum){
key = key +"##"+unitNum
}
if(!keys.contains(key)){
keys << key
def obj = findLoudongOnLine(d.city, newRegion, d.district_name, house_num, d.unit_num)
println d.district_name+" "+d.house_num+"==${house_num}==${unitNum}"
if(obj){
def udata = new BasicDBObject()
def areas = obj.areas
if(!areas.contains(d.area)){
areas << d.area
if(detailMap.height){
def forms = obj.from
if(!forms.contains("房谱网")){
forms << "房谱网"
}
obj.m_date = new Date()
def query = new BasicDBObject("_id", obj._id)
LOUDONG_LINE_COLL.update(query, new BasicDBObject('$set', obj))
}
}else{
def udata = new BasicDBObject()
udata.put("city", d.city)
udata.put("region", newRegion)
udata.put("name", d.district_name)
udata.put("house_num", house_num)
if(unitNum){
udata.put("unit_num", unitNum)
}
if(d.address){
def address = AddressUtil.formatAddress(d.city, d.address)
if(address){
udata.put("address", address)
}
}
udata.put("areas",[d.area])
udata.put("from",["房谱网"])
if(d.floor_heigth && d.floor_heigth.toString().toInteger()){
udata.put("floors", d.floor_heigth.toString().toInteger())
}
udata.put("c_date", new Date())
udata.put("status", 1)
LOUDONG_LINE_COLL.save(udata)
}
}
}
}
}
return rets;
}
def addFdcHouseLoudong(){
def q = new BasicDBObject()
def dbcursor = loudongFdcConn.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(2000)
def keys = []
def index = 0
for(def d : dbcursor){
index++
println index
if(!d.use_type){
continue
}
if(!d.use_type.contains("住宅")){
continue
}
def newRegion = d.match_region
if(newRegion){
def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num_desc, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?")
def house_num = null
if(houseNumStr){
house_num = houseNumStr.replace("号楼","").replace("幢","").replace("栋","").replace("号","")
}
if(house_num){
def key = d.city +"##"+newRegion+"##"+d.name+"##"+house_num+"##"+d.unit_num+"##"+d.room_num
if(!keys.contains(key)) {
keys << key
def count = findHouseLoudongDatas(d.city, newRegion, d.name, house_num, d.unit_num, d.room_num)
if(!count){
def maxHeight = findMaxHeight(d.city, d.name, d.house_num_desc, d.unit_num)
def dist = null
if(d.match_city){
dist = CommonDatautils.findDistrict(d.match_city, d.match_region, d.match_name,"district")
}
def floor = null
if(d.room_num && d.room_num.contains("-")) {
def arrs = d.room_num.split("-")
floor = arrs[0]
}
Integer room = null
Integer hall = null
if(d.room_type && d.room_type.contains("室")){
def roomStr = d.room_type.substring(0,d.room_type.indexOf("室"))
if(roomStr){
roomStr = roomStr.replace("一","1").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("六","6").replace("七","7").replace("八","8").replace("九","9").replace("十","10")
room = roomStr.toInteger()
}
if(d.room_type.contains("厅")){
def hallStr = d.room_type.substring(d.room_type.indexOf("室")+1, d.room_type.indexOf("厅"))
if(hallStr){
hallStr = hallStr.replace("一","1").replace("两","2").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("六","6").replace("七","7").replace("八","8").replace("九","9").replace("十","10")
hall = hallStr.toInteger()
}
}
}
def udata = new BasicDBObject()
udata.put("city", d.city)
udata.put("region", newRegion)
udata.put("name", d.name)
udata.put("house_num", house_num)
if(d.unit_num){
udata.put("house_num_unit", d.unit_num)
}
udata.put("room_num", d.room_num)
udata.put("house_area",d.area.toDouble())
udata.put("status", 0)
udata.put("from", "重庆网上房地产")
udata.put("house_height", maxHeight)
if(room){
udata.put("room", room)
}
if(hall){
udata.put("hall", hall)
}
if(floor && floor.isNumber()){
udata.put("house_floor", floor.toInteger())
}
if(dist){
udata.put("district_id", dist._id.toString())
udata.put("name_in_seaweed", dist.name)
}
if(d.address){
def address = AddressUtil.formatAddress(d.city, d.address)
if(address){
udata.put("address", address)
}
}
udata.put("c_date", new Date())
udata.put("m_date", new Date())
HOUSE_LOUDONG_LINE_COLL.save(udata)
}
}
}
}
}
}
public String getMatchString(String text,String pattern){
Pattern p=Pattern.compile(pattern);
Matcher m=p.matcher(text);
if(m.find()){
return m.group(0)
}
return null;
}
def addFdcBaseLoudong(){
def q = new BasicDBObject()
def dbcursor = loudongFdcConn.find(q).batchSize(5000)
def keys = []
for(def d : dbcursor){
if(!d.match_region){
continue
}
def newRegion = d.match_region
def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num_desc, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?")
def house_num = null
if(houseNumStr){
house_num = houseNumStr.replace("号楼","").replace("幢","").replace("栋","").replace("号","")
}
if(house_num){
def key = d.city +"##"+newRegion+"##"+d.name+"##"+house_num
if(d.unit_num) {
key = key + "##" + d.unit_num
if (!keys.contains(key)) {
keys << key
def obj = findLoudongOnLine(d.city, newRegion, d.name, house_num, d.unit_num)
println d.name + " " + d.house_num_desc + "==${house_num}==${d.unit_num}"
def maxHeight = findMaxHeight(d.city, d.name, d.house_num_desc, d.unit_num)
if (obj) {
def areas = obj.areas
if (!areas.contains(d.area)) {
areas << d.area
def forms = obj.from
if (!forms.contains("重庆网上房地产")) {
forms << "重庆网上房地产"
}
obj.m_date = new Date()
def query = new BasicDBObject("_id", obj._id)
LOUDONG_LINE_COLL.update(query, new BasicDBObject('$set', obj))
}
} else {
def udata = new BasicDBObject()
udata.put("city", d.city)
udata.put("region", newRegion)
udata.put("name", d.name)
udata.put("house_num", house_num)
if (d.unit_num) {
udata.put("unit_num", d.unit_num)
}
if (d.address) {
def address = AddressUtil.formatAddress(d.city, d.address)
if (address) {
udata.put("address", address)
}
}
udata.put("areas", [d.area])
udata.put("from", ["重庆网上房地产"])
udata.put("floors", maxHeight)
udata.put("c_date", new Date())
udata.put("status", 1)
LOUDONG_LINE_COLL.save(udata)
}
}
}
}
}
}
def findMaxHeight(city, name,house_num, unit_num){
def q = new BasicDBObject()
q.put("city",city)
q.put("name",name)
q.put("house_num_desc",house_num)
q.put("unit_num",unit_num)
def dbcursor = loudongFdcConn.find(q)
def heights = []
for(def h : dbcursor){
if(h.nominal_floor && h.nominal_floor.isNumber()){
def height = h.nominal_floor.toInteger()
if(!heights.contains(height)){
heights << height
}
}
}
if(heights){
heights = heights.sort()
return heights[heights.size()-1]
}
return null
}
def findLoudongOnLine(city, region, name, house_num, unit_num){
def q = new BasicDBObject()
q.put("city",city)
q.put("region",region)
q.put("name",name)
q.put("house_num",house_num)
if(unit_num){
q.put("unit_num",unit_num)
}
def dbo = LOUDONG_LINE_COLL.findOne(q)
return dbo
}
def stat(){
def q = new BasicDBObject()
@@ -291,7 +684,51 @@ class CqfdcMatch {
}
def addLoudong(){
// def stat = statBaseLoudong()
// println stat
// findAllLoundongByHouseNum();
def stat1 = statHouseLoudong()
println stat1
// addHouseLoudong()
}
def addFdcLoudong(){
// addFdcBaseLoudong()
addFdcHouseLoudong()
}
def statHouseLoudong(){
def q = new BasicDBObject()
q.put("city", "重庆")
def dbcursor = HOUSE_LOUDONG_LINE_COLL.find(q)
def total = dbcursor.size()
def ids = []
for(def d : dbcursor){
if(d.district_id && !ids.contains(d.district_id)){
ids << d.district_id
}
}
def map = [:]
map.rows_total = total
map.dist_count = ids.size()
return map
}
def statBaseLoudong(){
def q = new BasicDBObject()
q.put("city", "重庆")
def dbcursor = LOUDONG_LINE_COLL.find(q)
def total = dbcursor.size()
def ids = []
for(def d : dbcursor){
if(d.id_in_seaweed && !ids.contains(d.id_in_seaweed)){
ids << d.id_in_seaweed
}
}
def map = [:]
map.rows_total = total
map.dist_count = ids.size()
return map
}
@@ -0,0 +1,28 @@
package com.fangjia.script.temp.shenjian.didata
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
/**
* 从excel中匹配出
*/
ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\地数\\数据建设\\待审核小区(地图匹配).xlsx", { row ->
def city = row['A']
def region = row['B']
def district_name = row['C']
def gaodeRegion = row['H']
def gaodeName = row['I']
def jsonObject = OpenApiClient.addressMatch(city, district_name, "district", false, region, district_name, "strict")
if (jsonObject != null && jsonObject.precise != -1) {
row['D'] = jsonObject.get("name")
}
if (gaodeRegion && gaodeName) {
def jsonObject2 = OpenApiClient.addressMatch(city, gaodeName, "district", false, gaodeRegion, gaodeName, "strict")
if (jsonObject2 != null && jsonObject2.precise != -1) {
row['J'] = jsonObject2.get("name")
}
}
});
@@ -0,0 +1,46 @@
package com.fangjia.script.temp.shenjian.geo
import com.fangjia.script.gj.location.Point
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.xh.InsidePolygon
import com.mongodb.BasicDBList
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "block", "192.168.0.85", 28017)
collection.find(new BasicDBObject()).forEach({ obj ->
if (obj.type == 'block') {
if (obj.region instanceof String) {
return;
}
BasicDBList reigon = (BasicDBList)obj.region
int count = 0
List<String> info = new ArrayList<>()
for (String s : reigon) {
info.add(s)
}
if (info.size() > 1) {
boolean find = false
for (String s : info) {
if (find) {
break
}
collection.find(new BasicDBObject("type", "region").append("name", s)).limit(1).forEach({obj2 ->
String border = obj2.border
String points = border.replaceAll(",", "-").replaceAll(";", "-")
List<com.fangjia.script.looplineinfo.Point> pointList = InsidePolygon.createPoint(points);
com.fangjia.script.looplineinfo.Point point = new com.fangjia.script.looplineinfo.Point(obj.lng, obj.lat)
if (InsidePolygon.aboutZone(point, pointList)) {
collection.update(new BasicDBObject("_id", obj._id),
new BasicDBObject('$set', new BasicDBObject("region", obj2.name)));
find = true;
}
})
}
} else {
collection.update(new BasicDBObject("_id", obj._id),
new BasicDBObject('$set', new BasicDBObject("region", info.get(0))));
}
}
})
@@ -2,6 +2,8 @@ package com.fangjia.script.temp.shenjian.poi
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.statistics.poi.AmapPoiDetail
import com.fangjia.script.temp.shenjian.util.CommonDtoUtils
import com.fangjia.script.temp.shenjian.util.ExcelUtils
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils
@@ -14,13 +16,15 @@ import java.util.concurrent.atomic.AtomicLong
class GetPoiFromExcel {
static final def URL = "http://192.168.0.216:38080/FangJiaDataService/saveAmapPoi"
static final ExecutorService executorService = Executors.newFixedThreadPool(10)
static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoiDetail"
// 住宅类
static final String typecodes = "120000|120100|120200|120201|120202|120203|120300|120301|120302|120303|120304";
static void main(String[] args) {
AtomicLong atomicLong = new AtomicLong(0)
//List<File> fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Desktop\\房价网\\项目\\建行\\模型级联\\POI"), null, false);
List<File> fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Desktop\\房价网\\项目\\建行\\模型级联\\高德反查无ID"), null, false);
List<File> fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Documents\\知识库\\建行\\审核小区"), null, false);
for (File file : fileList) {
List<List<String>> seaweeds = ExcelUtils.read(file.getPath())
for (List<String> infoTemp : seaweeds) {
@@ -31,9 +35,10 @@ class GetPoiFromExcel {
String city = info.get(0)
String region = info.get(1)
String seaweedName = info.get(2)
List<AmapPoi> pois = GaoDeUtils.searchPoi(city, seaweedName, region, "120300|120301|120302");
List<AmapPoi> pois = GaoDeUtils.searchPoi(city, seaweedName, region, typecodes);
if (CollectionUtils.isNotEmpty(pois)) {
String jsonStr = JSONObject.toJSONString(pois)
List<AmapPoiDetail> detailList = CommonDtoUtils.transformListV(AmapPoiDetail.class, pois);
String jsonStr = JSONObject.toJSONString(detailList)
HttpUtils.postForJson(URL, jsonStr)
atomicLong.incrementAndGet()
}
@@ -13,7 +13,13 @@ import com.fangjia.script.wenyan.auto.KeywordSearchUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.client.model.Filters
import com.mongodb.client.model.UpdateOneModel
import com.mongodb.client.model.UpdateOptions
import com.mongodb.client.model.WriteModel
import org.apache.commons.collections.CollectionUtils
import org.bson.Document
import org.bson.conversions.Bson
import org.bson.types.ObjectId
import java.util.concurrent.ExecutorService
@@ -27,74 +33,85 @@ import java.util.concurrent.Executors
*/
class SeaweedAroundPoiSingal {
static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017)
static def conn136 = MongoUtils.getCollectionByName3("fangjia", "seaweed_poi_sync", "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017")
static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi"
static void main(String[] args) {
def types = listAllType()
//def types = "0601"
DBObject query = new BasicDBObject()
query.put("status", 1)
conn136.find(query).batchSize(30).each {objTemp ->
final def obj = objTemp
def city = obj.city
def lng = obj.lng
def lat = obj.lat
List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "")
if (CollectionUtils.isNotEmpty(pois)) {
String jsonStr = JSONObject.toJSONString(pois)
HttpUtils.postForJson(URL, jsonStr)
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI")))
} else {
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空")))
}
println("已处理小区:" + obj)
}
getNewSeaweed()
// def types = listAllType()
// //def types = "0601"
// DBObject query = new BasicDBObject()
// query.put("status", 1)
// conn136.find(query).batchSize(30).each {objTemp ->
// final def obj = objTemp
// def city = obj.city
// def lng = obj.lng
// def lat = obj.lat
// List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "")
// if (CollectionUtils.isNotEmpty(pois)) {
// String jsonStr = JSONObject.toJSONString(pois)
// HttpUtils.postForJson(URL, jsonStr)
// conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI")))
// } else {
// conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空")))
// }
// println("已处理小区:" + obj)
// }
}
static getNewSeaweed() {
DBObject query = new BasicDBObject()
query.put("cat", "district")
query.put("visible", 0)
//query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a"))
DBObject filter = new BasicDBObject()
filter.put("_id", 1)
filter.put("city", 1)
filter.put("region", 1)
filter.put("name", 1)
filter.put("alias", 1)
filter.put("address", 1)
filter.put("addr_alias", 1)
filter.put("lng", 1)
filter.put("lat", 1)
List list = []
List<WriteModel<Document>> writeModelList = new ArrayList<>()
def count = 0
DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj ->
if (conn136.count(new BasicDBObject("_id", obj._id)) > 0) {
// 已存在小区,无需同步
} else {
DBObject newObj = new BasicDBObject()
newObj.put("_id", obj._id)
newObj.put("city", obj.city)
newObj.put("region", obj.region)
newObj.put("name", obj.name)
try {
newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng))
newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat))
} catch (Exception ex) {
print("经纬度为空:" + obj._id + " " + obj)
ex.printStackTrace()
}
newObj.put("status", 1)
newObj.put("flag", "未获取周边POI")
Document updateBson = new Document()
Document update = new Document()
update.put("city", obj.city)
update.put("region", obj.region)
update.put("name", obj.name)
update.put("alias", obj.alias)
update.put("address", obj.address)
update.put("addr_alias", obj.addr_alias)
try {
update.put("lng", SeaweedUtils.formatLatLng(obj.lng))
update.put("lat", SeaweedUtils.formatLatLng(obj.lat))
} catch (Exception ex) {
print("经纬度为空:" + obj._id + " " + obj)
}
update.put("status", 1)
update.put("flag", "未精确获取高德信息")
updateBson.append('$set', update)
list << newObj
if (list.size() == 5000) {
conn136.insert(list)
list.clear()
}
Bson filterBson = Filters.eq("_id", obj._id)
UpdateOptions updateOptions = new UpdateOptions();
updateOptions.upsert(true)
UpdateOneModel updateOneModel = new UpdateOneModel(filterBson, updateBson, updateOptions);
writeModelList.add(updateOneModel)
if (writeModelList.size() == 5000) {
conn136.bulkWrite(writeModelList)
count += 5000
println(count)
writeModelList.clear()
}
}
conn136.insert(list)
list.clear()
if (writeModelList.size() > 0) {
conn136.bulkWrite(writeModelList)
writeModelList.clear()
}
}
static def getByIdAndInsert(String id) {
@@ -0,0 +1,163 @@
package com.fangjia.script.temp.shenjian.poi
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.statistics.poi.AmapPoiDetail
import com.fangjia.script.temp.shenjian.util.CommonDtoUtils
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.wenyan.auto.KeywordSearchUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.client.model.Filters
import com.mongodb.client.model.UpdateOneModel
import com.mongodb.client.model.UpdateOptions
import com.mongodb.client.model.WriteModel
import org.apache.commons.collections.CollectionUtils
import org.apache.commons.lang.StringUtils
import org.bson.Document
import org.bson.conversions.Bson
import org.bson.types.ObjectId
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 小区POI信息精确查找
*
* @author shenjian
* @since 2022/5/25
*/
class SeaweedPoiDetail {
static def conn136 = MongoUtils.getCollectionByName3("fangjia", "seaweed_poi_sync", "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017")
static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoiDetail"
// 住宅类
static final String typecodes = "120000|120100|120200|120201|120202|120203|120300|120301|120302|120303|120304";
static final ExecutorService executorService = Executors.newFixedThreadPool(20)
static void main(String[] args) {
DBObject query = new BasicDBObject()
query.put("status", 1)
conn136.find(query).batchSize(1000).each {objTemp ->
final def obj = objTemp
executorService.submit(new Runnable() {
@Override
void run() {
def city = obj.city
def region = obj.region
def name = obj.name
def alias = obj.alias
def address = obj.address
def addr_alias = obj.addr_alias
StringJoiner stringJoiner = new StringJoiner("|")
// 主小区名搜索
stringJoiner.add(name)
// 根据别名搜
if (alias != null && CollectionUtils.isNotEmpty(alias)) {
for (String str : alias) {
stringJoiner.add(str)
}
}
// 根据路名搜
if (StringUtils.isNotBlank(address)) {
stringJoiner.add(address)
}
// 根据路别名搜
if (addr_alias != null && CollectionUtils.isNotEmpty(addr_alias)) {
for (String str : addr_alias) {
stringJoiner.add(str)
}
}
String keywords = stringJoiner.toString()
List pois = GaoDeUtils.searchPoi(city, keywords, region, typecodes)
if (CollectionUtils.isEmpty(pois)) {
pois = GaoDeUtils.searchPoi(city, name, region, typecodes)
}
if (CollectionUtils.isEmpty(pois)) {
pois = GaoDeUtils.searchPoi(city, address, region, typecodes)
}
List<AmapPoiDetail> detailList = CommonDtoUtils.transformListV(AmapPoiDetail.class, pois);
// for (AmapPoi amapPoi : pois) {
// List<AmapPoiDetail> amapPoiList = GaoDeUtils.getAmapDetailById(amapPoi.getId());
// if (CollectionUtils.isNotEmpty(amapPoiList)) {
// detailList.addAll(amapPoiList)
// }
// }
if (CollectionUtils.isNotEmpty(detailList)) {
String jsonStr = JSONObject.toJSONString(detailList)
HttpUtils.postForJson(URL, jsonStr)
conn136.updateOne(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取到")))
} else {
conn136.updateOne(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "未获取到小区")))
}
println("已处理小区:" + obj)
}
})
}
executorService.shutdown()
}
static getNewSeaweed() {
DBObject query = new BasicDBObject()
query.put("cat", "district")
query.put("visible", 0)
DBObject filter = new BasicDBObject()
filter.put("_id", 1)
filter.put("city", 1)
filter.put("region", 1)
filter.put("name", 1)
filter.put("alias", 1)
filter.put("address", 1)
filter.put("addr_alias", 1)
filter.put("lng", 1)
filter.put("lat", 1)
List<WriteModel<Document>> writeModelList = new ArrayList<>()
def count = 0
DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj ->
Document updateBson = new Document()
Document update = new Document()
update.put("city", obj.city)
update.put("region", obj.region)
update.put("name", obj.name)
update.put("alias", obj.alias)
update.put("address", obj.address)
update.put("addr_alias", obj.addr_alias)
try {
update.put("lng", SeaweedUtils.formatLatLng(obj.lng))
update.put("lat", SeaweedUtils.formatLatLng(obj.lat))
} catch (Exception ex) {
print("经纬度为空:" + obj._id + " " + obj)
}
update.put("status", 1)
update.put("flag", "未精确获取高德信息")
updateBson.append('$set', update)
Bson filterBson = Filters.eq("_id", obj._id)
UpdateOptions updateOptions = new UpdateOptions();
updateOptions.upsert(true)
UpdateOneModel updateOneModel = new UpdateOneModel(filterBson, updateBson, updateOptions);
writeModelList.add(updateOneModel)
if (writeModelList.size() == 5000) {
conn136.bulkWrite(writeModelList)
count += 5000
println(count)
writeModelList.clear()
}
}
if (writeModelList.size() > 0) {
conn136.bulkWrite(writeModelList)
writeModelList.clear()
}
}
}
@@ -0,0 +1,40 @@
package com.fangjia.script.temp.shenjian.util;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
/**
* 通用实体与DTO互相转换工具类
*
* @author shenjian
* @date 2021/06/16
*/
public class CommonDtoUtils {
public static <T> T transformV(Class<T> targetClass, Object source) {
if (source == null) {
return null;
} else {
try {
String jsonSource = JSON.toJSONString(source);
return JSONObject.parseObject(jsonSource, targetClass);
} catch (Exception var3) {
throw var3;
}
}
}
public static <T> List<T> transformListV(Class<T> targetClass, List<?> listSource) {
if (listSource == null) {
return null;
} else {
try {
String jsonSource = JSON.toJSONString(listSource);
return JSONArray.parseArray(jsonSource, targetClass);
} catch (Exception var3) {
throw var3;
}
}
}
}
@@ -7,10 +7,12 @@ import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.statistics.poi.AmapPoiDetail
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.HttpRequestUtil
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import org.apache.commons.lang.StringUtils
import org.openqa.selenium.json.Json
import java.util.concurrent.atomic.AtomicInteger
@@ -88,12 +90,25 @@ class GaoDeUtils {
if (json && json.info && json.info == "OK") {
count = json.getString("count").toInteger()
result.addAll(DataUtil.toMongoDbObj(json.pois))
} else if (json && json.infocode && json.infocode == 10003) {
throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封")
} else if (json && json.infocode && json.infocode == 10004) {
} else if (json && json.infocode && json.infocode == "10044") {
synchronized (this) {
for (int i = 0; i < GAO_DE_AK.size(); i++) {
if (GAO_DE_AK[i] == params.key) {
if (i + 1 >= GAO_DE_AK.size()) {
System.exit(1)
}
keyUsed = new AtomicInteger(i+1)
}
}
break;
}
} else if (json && json.infocode && json.infocode == "10004") {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000)
break;
} else if (json && json.infocode && json.infocode == "10017") {
// 所请求的资源不可用
System.exit(1)
} else {
break
}
@@ -146,7 +161,7 @@ class GaoDeUtils {
synchronized (this) {
for (int i = 0; i < GAO_DE_AK.size(); i++) {
if (GAO_DE_AK[i] == params.key) {
if (i + 1 > GAO_DE_AK.size()) {
if (i + 1 >= GAO_DE_AK.size()) {
System.exit(1)
}
keyUsed = new AtomicInteger(i+1)
@@ -160,7 +175,9 @@ class GaoDeUtils {
break;
} else if (json && json.infocode && json.infocode == "10017") {
// 所请求的资源不可用
System.exit(1)
//System.exit(1)
Thread.sleep(60000)
break;
} else {
break
}
@@ -222,12 +239,65 @@ class GaoDeUtils {
} catch(Exception ex) {
ex.printStackTrace()
}
} else if (json && json.infocode && json.infocode == "10044") {
synchronized (this) {
for (int i = 0; i < GAO_DE_AK.size(); i++) {
if (GAO_DE_AK[i] == params.key) {
if (i + 1 >= GAO_DE_AK.size()) {
System.exit(1)
}
keyUsed = new AtomicInteger(i+1)
}
}
}
} else if (json && json.infocode && json.infocode == "10004") {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000)
} else if (json && json.infocode && json.infocode == "10017") {
// 所请求的资源不可用
System.exit(1)
//System.exit(1)
Thread.sleep(60000)
}
return Collections.emptyList()
}
/**
* 获取详情
*/
static List<AmapPoiDetail> getAmapDetailById(id){
def params = [:]
params.key = GAO_DE_AK[keyUsed.intValue()]
params.id = id
params.output = 'json'
JSONObject json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v3/place/detail", params);
if (json == null) {
json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v3/place/detail", params);
}
if (json && json.info && json.info == "OK") {
try {
List<AmapPoiDetail> ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class)
return ret
} catch(Exception ex) {
ex.printStackTrace()
}
} else if (json && json.infocode && json.infocode == "10044") {
synchronized (this) {
for (int i = 0; i < GAO_DE_AK.size(); i++) {
if (GAO_DE_AK[i] == params.key) {
if (i + 1 >= GAO_DE_AK.size()) {
System.exit(1)
}
keyUsed = new AtomicInteger(i+1)
}
}
}
} else if (json && json.infocode && json.infocode == "10004") {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000)
} else if (json && json.infocode && json.infocode == "10017") {
// 所请求的资源不可用
//System.exit(1)
Thread.sleep(60000)
}
return Collections.emptyList()
}
@@ -270,7 +340,7 @@ class GaoDeUtils {
synchronized (this) {
for (int i = 0; i < GAO_DE_AK.size(); i++) {
if (GAO_DE_AK[i] == params.key) {
if (i + 1 > GAO_DE_AK.size()) {
if (i + 1 >= GAO_DE_AK.size()) {
System.exit(1)
}
keyUsed = new AtomicInteger(i+1)
@@ -278,13 +348,13 @@ class GaoDeUtils {
}
break;
}
} else if (json && json.infocode && json.infocode == "10004") {
} else if (json && json.infocode && (json.infocode == "10004" || json.infocode == "10021" || json.infocode == "10017")) {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000)
break;
} else if (json && json.infocode && json.infocode == "10017") {
// 所请求的资源不可用
System.exit(1)
} else if (json && json.infocode && (json.infocode == "20000")) {
// 无效参数
break;
}
}
return result
@@ -326,7 +396,7 @@ class GaoDeUtils {
synchronized (this) {
for (int i = 0; i < GAO_DE_AK.size(); i++) {
if (GAO_DE_AK[i] == params.key) {
if (i + 1 > GAO_DE_AK.size()) {
if (i + 1 >= GAO_DE_AK.size()) {
System.exit(1)
}
keyUsed = new AtomicInteger(i+1)
@@ -367,7 +437,16 @@ class GaoDeUtils {
return ["lng": arrs[0], "lat": arrs[1]]
}
} else if (json && json.infocode && json.infocode == "10044") {
System.exit(1)
synchronized (this) {
for (int i = 0; i < GAO_DE_AK.size(); i++) {
if (GAO_DE_AK[i] == params.key) {
if (i + 1 >= GAO_DE_AK.size()) {
System.exit(1)
}
keyUsed = new AtomicInteger(i+1)
}
}
}
} else if (json && json.infocode && json.infocode == "10004") {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000)
@@ -128,10 +128,10 @@ class SeaweedUtils {
*/
static formatHouseholdCount(def household_count) {
if (household_count && FieldValidatorUtils.checkValue(household_count) ) {
if (Long.valueOf(household_count) < 50) {
if (Integer.valueOf(household_count) < 50) {
return "" // »§Êý
}
return household_count
return household_count.toInteger()
}
}
@@ -514,6 +514,7 @@ class DataStandardBatch extends BaseBatch{
def filterSpacialChar(def s){
def pattern = ~/[()!;\·]\[【】()]/
def matcher = pattern.matcher(s)
def count = matcher.getCount()
if(count && count>0){
return matcher.replaceAll('_')
@@ -43,8 +43,8 @@ class DataStat {
def source = "58ͬ³Ç"//
def citys = findCrawlCity(source, 202202);
for(def city : citys){
def count1 = findCount(source, city, 202202);
def count2 = findCount(source, city, 202203);
def count1 = findCount(source, city, 202203);
def count2 = findCount(source, city, 202204);
println city +" "+count1+" "+count2
// println count1
// println "--- ----"
@@ -40,13 +40,13 @@ class Esf58Crawl extends BaseBatch{
def static useDistKeys = []
static def totalCount = new AtomicLong()
def cityList = ["成都":"cd|102","杭州":"hz|79","重庆":"cq|37","苏州":"su|5","无锡":"wx|93", "深圳":"sz|4","郑州":"zz|342", "长沙":"cs|414","东莞":"dg|413",
def cityList = ["福州":"fz|304","成都":"cd|102","杭州":"hz|79","重庆":"cq|37","苏州":"su|5","无锡":"wx|93", "深圳":"sz|4","郑州":"zz|342", "长沙":"cs|414","东莞":"dg|413",
"괏땍":"bd|424","괏癸":"baoting|10367","켓퀼":"nn|845", "爛鹵":"yz|637","�庫":"st|783","揭鹵":"taizhou|693","뺨갛":"ha|968","纜쉭":"zhanjiang|791","쏩鹵":"jinzhou|2354","웹츠":"km|541","진鹵":"liuzhou|7133","凉헤":"zq|901","젯暾멍":"lyg|2049","휑亮":"rizhao|3177","끽돠":"changde|872",
"꾕鹵":"cangzhou|652", "홋鹵":"qz|291","헌陶":"qingyuan|7303", "鮫퀼":"xn|2052","潾屢":"zunyi|7620","瓏碌":"zhuzhou|1086","뱍捺":"gy|2015","纛捺":"yy|821","其鹵":"xz|471", "벗랑깝":"hrb|202","露끗":"yc|858", "읜렌":"lf|772","해뽈돎":"qhd|1078","쁨鹵":"huizhou|722","쭤捺":"luoyang|556",
"柯소鏤":"sjz|241","格覩":"ty|740","吉냘":"yancheng|613","斤捺":"xy|8694","켓념":"nanchong|2378","켓捺":"ny|592","衢쳔":"xm|606",
"축捺":"mianyang|1057","셌퀼":"jining|450","벳덱":"hd|572",
"뺨굇":"huaibei|9357","뇔鹵":"chuzhou|10266",
"福州":"fz|304","蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158",
"蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158",
"읊鹵":"lz|952","북례":"hf|837", "魁쨀컁펭":"xj|984","켓끗":"nc|669","�捺":"sy|188","鮫갛":"xa|483","藿렌":"wf|362",
"交빤":"wuhu|2045","땜賻":"dg|413","끽鹵":"cz|463","뤘�":"fs|222",
"켓繫":"nt|394","섈주":"jl|700","櫓�":"zs|771","막捺":"fy|2325",
@@ -192,7 +192,8 @@ class Esf58Crawl extends BaseBatch{
void process(Object args) {
//addContent("르鹵","https://fz.58.com/xiaoqu/", 70);
def citys = ["北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏"]
// "宁波","佛山","南通","烟台","临沂","呼和浩特","海口","广州",
def citys = ["唐山","泰安","襄阳","台州","镇江","银川","江门","荆州","北京","衡阳","南京","滨州","张家口","商丘","赣州","湖州","安庆","聊城","枣庄","阜阳","西安","芜湖","吉林","遂宁","鞍山","东营","安阳","咸阳"]
for(def city : citys){
def str = cityList.get(city)
for(def i=0;i<1;i++){
@@ -18,7 +18,7 @@ class SynchDistrictPrice {
// def sourceMap = ["fang.com":"搜房","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"]
//"fang.com":"搜房",
def sourceMap = ["fang.com":"搜房","ke.com":"贝壳找房","5i5j.com":"我爱我家"]
def sourceMap = ["ke.com":"贝壳找房","5i5j.com":"我爱我家"]
static main(args) {
def task = new SynchDistrictPrice()
task.run()
@@ -26,7 +26,7 @@ class SynchDistrictPrice {
def run(){
def citys = CommonUtils.findAllCity()
def citys = ["重庆"]//CommonUtils.findAllCity()
sourceMap.each { k, v ->
for(def city : citys){
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
for(def source :sources){
def count1 = findTotalCount(source, 202202);
def count2 = findTotalCount(source, 202203);
def count1 = findTotalCount(source, 202203);
def count2 = findTotalCount(source, 202204);
println source +" "+count1 +" "+count2
}
}
@@ -169,7 +169,7 @@ public class CommonUtils {
static void sendMessage(String queueName, byte[] bytes){
ConnectionFactory factory = new ConnectionFactory();
factory.setUri("amqp://guest:guest@192.168.0.19:45672/%2f");
factory.setUri("amqp://guest:guest@192.168.0.100:35677/%2f");
Connection conn = factory.newConnection();
Channel channel = conn.createChannel();
channel.queueDeclare(queueName, true, false, false, null);
@@ -92,11 +92,6 @@ public class DbUtils{
}
}
public static void executeQueryOnSeawater(BasicDBObject query,Closure c, BasicDBObject fields = null){
seawater.find(query,fields).batchSize(10000).each(c)
}
public static void executeQueryOnSeaweed(BasicDBObject query,Closure c, BasicDBObject fields = null){
SEAWEED_SLAVE.find(query,fields).batchSize(1000).each(c)
}
@@ -171,10 +166,6 @@ public class DbUtils{
return SEAWEED_SLAVE.findOne(query,fields);
}
public static int executeCountOnSeawater(BasicDBObject query){
return seawater.count(query);
}
public static BasicDBObject buildSeaweedQuery(def city = null, def region = null, def name = null, def cat = "district", def visible = 0){
BasicDBObject query = new BasicDBObject()
if(city){
@@ -387,31 +378,6 @@ public class DbUtils{
return district.lat && district.lng && district.lng > 0 && district.lat > 0
}
/**
* 记录修改小区库的操作日志
* @param city
* @param region
* @param name
* @return
*/
def static saveDeleteLog(String city, String region,String name,String category,int status){
if(status!=1){
return
}
DBObject obj = new BasicDBObject()
obj.put("city", city);
obj.put("region", region);
obj.put("district_name", name);
obj.put("category" , category)
obj.put("origin" , "脚本")
if(status==1){
obj.put("type" , "remove")
}
obj.put("c_date" , new Date())
obj.put("m_date" , new Date())
OPERATION_LOG.save(obj)
}
public static DBCollection getCollection(String host, int port, String dbName ,String collName){
def collection = COLL_CACHE["${host}#${port}#${dbName}#${collName}"]
if(!collection){
@@ -229,6 +229,153 @@ public class OpenApiClient {
return null
}
public static JSONObject searchLoudong(String id, String houseNum, String unitNum){
try{
Map<String, String> params = new HashMap<String, String>();
params.put("id", id);
params.put("houseNumber", houseNum);
params.put("unitNumber", unitNum);
params.put("token", TOKEN);
String content = new HttpRequest().get("http://open.fangjia.com/property/searchLoudong", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
while (result == null) {
result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/property/searchLoudong", params, "utf-8")
println("${id} ${houseNum} 程序调用异常")
Thread.sleep(6000)
}
if(result){
if (result.getInt("code") == 200) {
return result;
} else if (result.getInt("code") !=0) {
println("${id} ${houseNum}")
}
}
} catch (Exception e) {
println("${id} ${houseNum} ${e.getMessage()}")
e.printStackTrace();
}
return null
}
public static JSONObject searchHouseLoudong(String id, String houseNum, String unitNum, String roomNum){
try{
Map<String, String> params = new HashMap<String, String>();
params.put("id", id);
params.put("houseNumber", houseNum);
params.put("unitNumber", unitNum);
params.put("roomNumber", roomNum);
params.put("token", TOKEN);
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/property/searchHouseLoudong", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
while (result == null) {
result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/property/searchHouseLoudong", params, "utf-8")
println("${id} ${houseNum} 程序调用异常")
Thread.sleep(6000)
}
if(result){
if (result.getInt("code") == 200) {
return result;
} else if (result.getInt("code") !=0) {
println("${id} ${houseNum}")
}
}
} catch (Exception e) {
println("${id} ${houseNum} ${e.getMessage()}")
e.printStackTrace();
}
return null
}
public static JSONObject webankEvaluate(String city, String region, String address, String area){
try{
Map<String, String> params = new HashMap<String, String>();
params.put("city", city);
params.put("district", region);
params.put("address", address);
params.put("area", area.toString());
params.put("token", TOKEN);
String content = new HttpRequest().get("http://open.fangjia.com/webank/evaluate", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
while (result == null) {
result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/webank/evaluate", params, "utf-8")
println("${city} ${address} 程序调用异常")
Thread.sleep(6000)
}
if(result){
if (result.getInt("code") == 200) {
return result;
} else if (result.getInt("code") !=0) {
println("${city} ${address} 程序调用异常")
}
}
} catch (Exception e) {
println("${city} ${address} ${e.getMessage()}")
e.printStackTrace();
}
return null
}
public static JSONObject standard2Action(String city, String address) {
try {
Map<String, String> params = new HashMap<String, String>();
params.put("city", city);
params.put("address", address);
params.put("token", TOKEN);
params.put("__refresh_cache","1")
String content = new HttpRequest().get("https://open.fangjia.com/v2/address/standard", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
while (result == null) {
result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/v2/address/standard", params, "utf-8")
println("${city} ${address} 程序调用异常")
Thread.sleep(6000)
}
if (result.getInt("code") == 0) {
return result.getJSONObject("result");
} else if (result.getInt("code") !=0) {
println("${city} ${address}")
}
} catch (Exception e) {
println("${city} ${address} ${e.getMessage()}")
e.printStackTrace();
}
return null
}
public static JSONObject standard1Action(String city, String address) {
try {
Map<String, String> params = new HashMap<String, String>();
params.put("city", city);
params.put("address", address);
params.put("token", TOKEN);
params.put("__refresh_cache","1")
String content = new HttpRequest().get("https://open.fangjia.com/address/standard", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
while (result == null) {
result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/address/standard", params, "utf-8")
println("${city} ${address} 程序调用异常")
Thread.sleep(6000)
}
if (result.getInt("code") == 200) {
return result.getJSONObject("result");
} else if (result.getInt("code") !=0) {
println("${city} ${address}")
}
} catch (Exception e) {
println("${city} ${address} ${e.getMessage()}")
e.printStackTrace();
}
return null
}
public static JSONObject standardize(String city, String address) {
try {
Map<String, String> params = new HashMap<String, String>();
@@ -593,6 +740,91 @@ public class OpenApiClient {
}
static JSONObject facilitys(String city, String region, String name) {
Map<String, String> params = new HashMap<String, String>();
params.put("token", TOKEN);
params.put("city", city)
params.put("region", region)
params.put("name", name)
try {
String content = new HttpRequest().get("http://open.fangjia.com/district/facilitys", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
return result
} catch (Exception e) {
}
return null
}
static JSONObject searchPaimais(String city, String region, String name, String startDate,limit) {
Map<String, String> params = new HashMap<String, String>();
params.put("token", TOKEN);
params.put("city", city)
params.put("district", region)
params.put("name", name)
params.put("startDate", startDate)
params.put("limit", "100")
try {
String content = new HttpRequest().get("http://open.fangjia.com/auction/searchPaimais", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
return result
} catch (Exception e) {
}
return null
}
static JSONObject searchDealPrice(String id, String startDate, limit) {
Map<String, String> params = new HashMap<String, String>();
params.put("token", TOKEN);
params.put("id", id)
params.put("startDate", startDate)
params.put("limit", limit.toString())
try {
String content = new HttpRequest().get("http://open.fangjia.com/dealPrice/searchList", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
return result
} catch (Exception e) {
}
return null
}
static JSONObject houseList(String id, String startDate, limit) {
Map<String, String> params = new HashMap<String, String>();
params.put("token", TOKEN);
params.put("id", id)
params.put("startTime", startDate)
params.put("pageSize", limit.toString())
try {
String content = new HttpRequest().get("http://open.fangjia.com/house/list", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
return result
} catch (Exception e) {
}
return null
}
static JSONObject evaluaionList(String id, String startDate, limit) {
Map<String, String> params = new HashMap<String, String>();
params.put("token", TOKEN);
params.put("id", id)
params.put("startDate", startDate)
params.put("limit", limit.toString())
try {
String content = new HttpRequest().get("http://open.fangjia.com/evaluation/bsList", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
return result
} catch (Exception e) {
}
return null
}
static JSONObject districtDetail(String city, String region, String name) {
Map<String, String> params = new HashMap<String, String>();
params.put("token", TOKEN);
@@ -0,0 +1,71 @@
package com.fangjia.script.statistics.poi;
import java.util.List;
/**
* @author shenjian
* @since 2022/5/25
*/
@lombok.NoArgsConstructor
@lombok.Data
public class AmapPoiDetail {
@com.alibaba.fastjson.annotation.JSONField(name = "id")
private String id;
@com.alibaba.fastjson.annotation.JSONField(name = "parent")
private List<?> parent;
@com.alibaba.fastjson.annotation.JSONField(name = "childtype")
private List<?> childtype;
@com.alibaba.fastjson.annotation.JSONField(name = "name")
private String name;
@com.alibaba.fastjson.annotation.JSONField(name = "type")
private String type;
@com.alibaba.fastjson.annotation.JSONField(name = "typecode")
private String typecode;
@com.alibaba.fastjson.annotation.JSONField(name = "biz_type")
private List<?> bizType;
@com.alibaba.fastjson.annotation.JSONField(name = "address")
private String address;
@com.alibaba.fastjson.annotation.JSONField(name = "location")
private String location;
@com.alibaba.fastjson.annotation.JSONField(name = "tel")
private String tel;
@com.alibaba.fastjson.annotation.JSONField(name = "pname")
private String pname;
@com.alibaba.fastjson.annotation.JSONField(name = "cityname")
private String cityname;
@com.alibaba.fastjson.annotation.JSONField(name = "adname")
private String adname;
@com.alibaba.fastjson.annotation.JSONField(name = "importance")
private List<?> importance;
@com.alibaba.fastjson.annotation.JSONField(name = "shopid")
private List<?> shopid;
@com.alibaba.fastjson.annotation.JSONField(name = "shopinfo")
private String shopinfo;
@com.alibaba.fastjson.annotation.JSONField(name = "poiweight")
private List<?> poiweight;
@com.alibaba.fastjson.annotation.JSONField(name = "distance")
private String distance;
@com.alibaba.fastjson.annotation.JSONField(name = "biz_ext")
private BizExtDTO bizExt;
@com.alibaba.fastjson.annotation.JSONField(name = "photos")
private List<PhotosDTO> photos;
@lombok.NoArgsConstructor
@lombok.Data
public static class BizExtDTO {
@com.alibaba.fastjson.annotation.JSONField(name = "rating")
private List<?> rating;
@com.alibaba.fastjson.annotation.JSONField(name = "cost")
private String cost;
}
@lombok.NoArgsConstructor
@lombok.Data
public static class PhotosDTO {
@com.alibaba.fastjson.annotation.JSONField(name = "title")
private List<?> title;
@com.alibaba.fastjson.annotation.JSONField(name = "url")
private String url;
}
}
@@ -1,12 +1,12 @@
package com.fangjia.service.address;
import com.fangjia.core.RegionData;
import com.fangjia.core.util.*;
import com.fangjia.core.util.DataUtil;
import com.fangjia.core.util.GeoUtils;
import com.fangjia.dal.QueryParam;
import com.fangjia.dal.common.DistrictKey;
import com.fangjia.dal.common.ServerAreaData;
import com.fangjia.dal.mongo.fangjia.model.ServerArea;
import com.fangjia.dal.mongo.model.AmapPoi;
import com.fangjia.dal.mongo.model.bkoff.HouseLoudong;
import com.fangjia.dal.mongo.model.bkoff.Loudong;
import com.fangjia.dal.seaweed.dao.SeaweedDao;
@@ -16,17 +16,15 @@ import com.fangjia.service.address.processor.AddressDistrictProcessors;
import com.fangjia.service.address.processor.AddressProcessors;
import com.fangjia.service.api.AliyunService;
import com.fangjia.service.bo.AliyunBo;
import com.fangjia.service.bo.MatchAddressScore;
import com.fangjia.service.datastore.DistrictService;
import com.fangjia.service.mongo.addrepo.AddressRepoService;
import com.fangjia.service.mongo.bkoff.HouseLoudongService;
import com.fangjia.service.mongo.bkoff.LoudongService;
import com.google.common.base.Function;
import com.google.common.collect.MapMaker;
import net.sf.json.JSONObject;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.bson.types.ObjectId;
import java.util.*;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
@@ -759,6 +757,13 @@ public class AddressParserServiceImpl implements AddressParserService {
double simScore;
boolean onSameRoad = false;
boolean sameDistrict = isSameDistrict(address, metaData);
if(PreciseLevel.STRICT == address.getPreciseLevel() && !sameDistrict){
return 0;
}
// //当地图解析出来的区域和用户地址中的区域一致时,只匹配指定区域下面的小区
// if(address.getGeoRegion() != null && address.getGeoRegion().equals(address.getDistrict()) && !sameDistrict){
// return 0;
// }
Iterator<Map.Entry<String, MatchText>> addrEntrySet = metaData.getAddrePinYinMap().entrySet().iterator();
Map.Entry<String, MatchText> addrEntry;
while(addrEntrySet.hasNext()){
@@ -808,7 +813,6 @@ public class AddressParserServiceImpl implements AddressParserService {
}
}
}
String keywordPinYin = address.getKeywordsPinYin();
String keywords = address.getKeywords();
String fuzzyKeywordsPinYin = address.getFuzzyKeywordsPinYin();
@@ -892,7 +896,7 @@ public class AddressParserServiceImpl implements AddressParserService {
if(name.length() == 2 && (name.equals(address.getOrgAddress()) || name.equals(address.getEffectiveAddress()))){
keywordScore = Math.max(keywordScore, namePriority);
}else if(name.length() > 3 && (address.getOrgAddrePinYin().contains(namePinYin) || enhanceContains(address.getOrgAddress(), name))){
double similar = AddressParserUtil.getSimilarityRatio(name, address.getEffectiveAddress());
double similar = AddressParserUtil.getSimilarityRatio(name, address.getOrgAddress());
if(similar == 1){
keywordScore = Math.max(similar, keywordScore) * namePriority * 1.0001;
}else if(similar > 0.1){
@@ -907,20 +911,27 @@ public class AddressParserServiceImpl implements AddressParserService {
if (standardAddressScore > 0 && keywordScore > 0) {
matchLevelScore = 0.9;
similarityScore = Math.max(standardAddressScore, keywordScore) * SIMILARITY_RATE;
} else if(keywordScore > 0 && streetScore > 0){
matchLevelScore = 0.8;
similarityScore = Math.max(streetScore, keywordScore * 0.8) * SIMILARITY_RATE;
} else if (standardAddressScore > 0) {
matchLevelScore = 0.7;
matchLevelScore = 0.8;
similarityScore = standardAddressScore * SIMILARITY_RATE;
} else if(keywordScore > 0 && streetScore > 0){
matchLevelScore = 0.7;
similarityScore = Math.max(streetScore, keywordScore * 0.8) * SIMILARITY_RATE;
} else if (keywordScore > 0) {
matchLevelScore = 0.6;
similarityScore = keywordScore * SIMILARITY_RATE;
} else if (fuzzyKeywordsScore > 0) {
matchLevelScore = 0.5;
if(address.hasCoordinate() && metaData.getLng() != null && GeoUtils.calculateDistance(address.getLng(), address.getLat(),
metaData.getLng(), metaData.getLat()) < 100){
matchLevelScore = 0.7;
}else{
matchLevelScore = 0.4;
}
similarityScore = fuzzyKeywordsScore * SIMILARITY_RATE;
}else if(metaData.getId().equals(address.getPolygonSeaweedId())){
matchLevelScore = 0.4;
similarityScore = 0.05;
}
if (similarityScore > 0) {
if(metaData.getBuildingType() != null){
if ("shop".equals(metaData.getBuildingType().toString())) {
@@ -942,10 +953,6 @@ public class AddressParserServiceImpl implements AddressParserService {
* similarityScore = ÏàËÆÈ²¿·Ö(0.07) + ÀàÐͲ¿·Ö(0.01) < 0.1
* */
double score = matchLevelScore + similarityScore;
if(address.hasCoordinate() && metaData.hasCoordinate()
&& GeoUtils.calculateDistance(address.getLng(), address.getLat(), metaData.getLng(), metaData.getLat()) > 3 * 1000){
score = 0d;
}
return score;
}
@@ -980,7 +987,7 @@ public class AddressParserServiceImpl implements AddressParserService {
return false;
}
if(StringUtils.isBlank(address.getDistrict()) && StringUtils.isBlank(address.getSuggestDistrict())){
return false;
return true;
}
String district = metaData.getDistrict();
return district.equals(address.getDistrict()) || district.equals(address.getSuggestDistrict());
@@ -61,6 +61,8 @@ public class Address {
private String suggestDistrict;
private String suggestName;
private String suggestNamePinYin;
private String polygonSeaweedId;
private String geoRegion;
private List<String> buildingNumbers;
private PreciseLevel preciseLevel;
private boolean pinYin;
@@ -750,6 +752,14 @@ public class Address {
return false;
}
public String getGeoRegion() {
return geoRegion;
}
public void setGeoRegion(String geoRegion) {
this.geoRegion = geoRegion;
}
@Override
public String toString() {
return "Address{" +
@@ -809,6 +819,14 @@ public class Address {
'}';
}
public String getPolygonSeaweedId() {
return polygonSeaweedId;
}
public void setPolygonSeaweedId(String polygonSeaweedId) {
this.polygonSeaweedId = polygonSeaweedId;
}
public String toMd5Hex(){
String keyProperties = "Address{" +
"city='" + city + '\'' +
@@ -20,18 +20,16 @@ public class AddressProcessors{
static {
//ÐÞ¸Ä˳ÐòÐèÒªµ÷Õûprocess i µÄÖµ
addressProcessors.add(new TokenProcessor());
addressProcessors.add(new ProximalCharactersProcessor());
addressProcessors.add(new PreNoiseProcessor());
addressProcessors.add(new AddressFormatProcessor());
addressProcessors.add(new TokenProcessor());
addressProcessors.add(new FloorProcessor());
addressProcessors.add(new TooltipProcessor());
addressProcessors.add(new ProvinceProcessor());
addressProcessors.add(new CityProcessor());
addressProcessors.add(new DistrictProcessor());
addressProcessors.add(new LoopLineProcessor());
addressProcessors.add(new TownshipProcessor());
addressProcessors.add(new LoopLineProcessor());
addressProcessors.add(new StageAndPartitionProcessor());
addressProcessors.add(new DirectionProcessor());
addressProcessors.add(new OffsetProcessor());
@@ -44,7 +42,7 @@ public class AddressProcessors{
addressProcessors.add(new NumericProcessor());
addressProcessors.add(new ProvinceAdjustProcessor());
addressProcessors.add(new FormatProcessor());
// addressProcessors.add(new GeoProcessor());
addressProcessors.add(new GeoProcessor());
}
@@ -52,8 +52,9 @@ public class CityProcessor extends AbstractAddressProcessor {
}
if(StringUtils.isNotBlank(city) && CityStatus.SURE != address.getCityStatus() ){
String formatCity = RegionData.formatCity(city);
String possibleCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) ? parseCityFromAddress(orgAddress, address) : null;
String confirmCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) ? null : formatCity;
address.setCity(formatCity);
String possibleCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) || address.getAddressParserGoal() == null ? parseCityFromAddress(orgAddress, address) : null;
String confirmCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) || address.getAddressParserGoal() == null? null : formatCity;
/**如果省份和城市信息相互矛盾 则用区域推断城市*/
if(StringUtils.isNotBlank(address.getProvince()) && !ProvinceData.belongTo(formatCity, address.getProvince())){
List<String> possibleCities = speculateCityNameByRegion(address, city);
@@ -191,15 +192,12 @@ public class CityProcessor extends AbstractAddressProcessor {
}else{
weight.score += 2;
}
}else if(address.contains(city)){
index = address.indexOf(city);
weight.index = index;
index += city.length();
if(province != null && (orgAddress.contains(province + city) || orgAddress.contains(ProvinceData.getDisplay(province) + city))){
weight.score += 2;
}else{
weight.score += 1;
}
}
Map<String, List<String>> regionMap = RegionData.getAllRegions(city);
@@ -208,9 +206,11 @@ public class CityProcessor extends AbstractAddressProcessor {
if(matchResult !=null){
address = address.replace(matchResult.matchContent, "");
}
if(address.indexOf(region, index) > -1){
if(address.indexOf(region + "Çø", index) > -1 || address.indexOf(region + "ÏØ", index) > -1){
if(province != null && (orgAddress.contains(province + region) || orgAddress.contains(ProvinceData.getDisplay(province) + region))){
weight.score += 2;
}else if(city.equals(addr.getCity())){
weight.score += 10;
}else{
weight.score += 1;
}
@@ -221,9 +221,13 @@ public class CityProcessor extends AbstractAddressProcessor {
List<String> alias = regionMap.get(region);
if(alias != null){
for(String alia : alias){
if(StringUtils.isNotBlank(alia) && !alia.equals(region) && address.indexOf(alia, index) > -1
if(StringUtils.isNotBlank(alia) && !alia.equals(region) && alia.length() > 2 && address.indexOf(alia, index) > -1
&& (alia.endsWith("市")|(alia.endsWith("县")))){
weight.score += 1;
if(city.equals(addr.getCity())){
weight.score += 10;
}else{
weight.score += 1;
}
weight.index = address.indexOf(alia, index);
break;
}
@@ -237,16 +241,6 @@ public class CityProcessor extends AbstractAddressProcessor {
}
List<FragmentWeight> weights = new ArrayList<FragmentWeight>(weightMap.values());
if(weights.size() > 0){
/*String street = getStreet(address);
if(street != null){
Iterator<FragmentWeight> iterator = weights.iterator();
while(iterator.hasNext()){
FragmentWeight fragmentWeight = iterator.next();
if(match(street, ".*" + fragmentWeight.text + "[东南西北中一二三四五六七八九十百千\\d]?(路|街|大道|支路|村)")){
iterator.remove();
}
}
}*/
Collections.sort(weights);
}
if(weights.size() > 0){
@@ -167,6 +167,9 @@ public class DistrictProcessor extends AbstractAddressProcessor {
address.setDevelopmentZone(dz);
}
}
if(address.getSuggestDistrict() != null && address.getDistrict() == null){
address.setDistrict(address.getSuggestDistrict());
}
address.setAddress(orgAddress);
address.setEffectiveAddress(address.getAddress());
return RepeatStatus.NEXT;
@@ -5,8 +5,12 @@ import com.fangjia.core.util.HttpRequest;
import com.fangjia.service.address.mode.Address;
import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
import org.apache.commons.lang.StringUtils;
import java.util.*;
import org.apache.log4j.Logger;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* µØÖ·¾­Î³È´¦ÀíÆ÷
@@ -16,78 +20,69 @@ import java.util.*;
public class GeoProcessor extends AbstractAddressProcessor {
private static final String KEY = "4ccfe3c898a9489884bd2bcb1ef6dcde";
List<String> POI_LEVELS = Arrays.asList("村庄", "热点商圈", "兴趣点", "门牌号", "单元号", "道路交叉路口");
List<String> DISTRICT_LEVEL = Arrays.asList("区县", "开发区", "乡镇", "村庄", "热点商圈", "兴趣点", "门牌号", "单元号", "道路交叉路口");
private static final Logger logger = Logger.getLogger(GeoProcessor.class);
List<String> POI_LEVELS = Arrays.asList("村庄", "热点商圈", "兴趣点", "门牌号", "单元号", "道路交叉路口", "门牌号");
@Override
public RepeatStatus process(Address address) {
if (address.hasCoordinate()) {
return RepeatStatus.NEXT;
}
List<String> queryTypes = new ArrayList<String>();
List<String> addressList = new ArrayList<String>();
if (address.isStandardAddress()) {
addressList.add(address.getStandardAddress());
queryTypes.add("StandardAddress");
}
if (StringUtils.isNotBlank(address.getOrgKeywords())) {
addressList.add(address.getOrgKeywords());
queryTypes.add("Keywords");
String addr = address.getEffectiveAddress();
if(addr == null || addr.length() < 3){
addr = address.getOrgAddress();
}
HttpRequest request = new HttpRequest();
Map<String, String> params = new HashMap<String, String>();
params.put("address", StringUtils.join(addressList, "|"));
params.put("address", addr);
params.put("key", KEY);
params.put("city", address.getCity());
params.put("batch", "true");
params.put("output", "json");
try {
JSONObject response = request.getJsonContent("https://restapi.amap.com/v3/geocode/geo", params, "UTF-8");
logger.info("高德API:" + response.toString());
if (response.getInt("status") == 1) {
JSONArray geocodes = response.getJSONArray("geocodes");
if(queryTypes.contains("StandardAddress")){
JSONObject geocode = geocodes.getJSONObject(queryTypes.indexOf("StandardAddress"));
String level = geocode.getString("level");
if(DISTRICT_LEVEL.contains(level)){
if(StringUtils.isBlank(address.getDistrict())){
address.setDistrict(RegionData.formatRegion(address.getCity(), geocode.getString("district")));
if(geocodes.size() > 0){
JSONObject json = geocodes.getJSONObject(0);
if(json.get("district") instanceof String){
String city = address.getCity();
if(city == null){
city = RegionData.formatCity(json.getString("city"));
}
String region = RegionData.formatRegion(city, json.getString("district"));
address.setGeoRegion(region);
}
if(geocode.containsKey("street") && geocode.containsKey("number")){
String geocodeAddress = geocode.getString("street") + geocode.getString("number");
if(address.getStandardAddress().contains(geocodeAddress)){
String location = geocode.getString("location");
address.setLng(Double.parseDouble(location.split(",")[0]));
address.setLat(Double.parseDouble(location.split(",")[1]));
}
}
}
if(queryTypes.contains("Keywords")){
JSONObject geocode = geocodes.getJSONObject(queryTypes.indexOf("Keywords"));
String level = geocode.getString("level");
if(DISTRICT_LEVEL.contains(level)){
if(StringUtils.isBlank(address.getDistrict())){
address.setDistrict(RegionData.formatRegion(address.getCity(), geocode.getString("district")));
}
}
if(POI_LEVELS.contains(level)){
String formattedAddress = geocode.getString("formatted_address");
formattedAddress = formattedAddress
.replaceFirst(geocode.getString("province"), "")
.replaceFirst(geocode.getString("city"), "")
.replaceFirst(geocode.getString("district"), "");
if(formattedAddress.equals(address.getOrgKeywords())){
String location = geocode.getString("location");
address.setLng(Double.parseDouble(location.split(",")[0]));
address.setLat(Double.parseDouble(location.split(",")[1]));
if(POI_LEVELS.contains(json.getString("level"))){
//对于门牌号级别的POI,formatted_address中包含|代表是推测出来的路号,重庆市巴南区凤林路|808号,|号楼
if("门牌号".equals(json.getString("level"))){
if(!(json.get("number") instanceof JSONArray) && json.getString("formatted_address").contains("|" + json.getString("number"))){
return RepeatStatus.NEXT;
}
}
String location = json.getString("location");
address.setLng(Double.valueOf(location.split(",")[0]));
address.setLat(Double.valueOf(location.split(",")[1]));
String tmp = "{ \"_source\": [\"seaweed_id\"], \"query\": {\"geo_shape\": {\"shape\": {\"shape\": {\"type\": \"Point\",\"coordinates\":[location]},\"relation\": \"CONTAINS\"}}}}";
tmp = tmp.replace("location", location);
Map<String, String> headers = new HashMap<String, String>();
headers.put("Authorization","Basic ZGV2ZWxvcGVyOjVlOGVkZDg1MWQyZmRmYmQ3NDE1MjMyYzY3MzY3Y2Mz");
JSONObject resp = JSONObject.fromObject(request.postWithBody("http://192.168.0.206:9200/seaweed_polygon/_search", JSONObject.fromObject(tmp),headers,"UTF-8"));
if(resp.containsKey("hits") && resp.getJSONObject("hits").containsKey("hits")){
JSONArray hits = resp.getJSONObject("hits").getJSONArray("hits");
if(hits.size() > 0){
String seaweedId = hits.getJSONObject(0).getJSONObject("_source").getString("seaweed_id");
address.setPolygonSeaweedId(seaweedId);
}
}
}
}
}
} catch (Exception e) {
logger.error(e);
e.printStackTrace();
}
return RepeatStatus.NEXT;
}
@@ -22,7 +22,7 @@ public class KeywordProcessor extends AbstractAddressProcessor {
static final List<Pattern> KEYWORDS_PATTERNS = Arrays.asList(Pattern.compile(".*[^一二三四五六七八九十\\d]{3,}号"));
static final List<String> KEYWORDS = Arrays.asList("居委会", "辖区", "街道办", "街道", "办事处", "村委", "管理区");
static final List<Pattern> NOISE_PATTERNS = Arrays.asList(Pattern.compile("^.*\\d+队"), Pattern.compile("^\\d+号院$"));
static final List<Pattern> INVALID_KEYWORD_PATTERN = Arrays.asList(Pattern.compile("^[一二三四五六七八九十甲乙丙丁戊己庚辛A-Z\\d]+[区期]$"), Pattern.compile("碧桂园"), Pattern.compile("住宅楼"), Pattern.compile("家属院"));
static final List<Pattern> INVALID_KEYWORD_PATTERN = Arrays.asList(Pattern.compile("^[一二三四五六七八九十甲乙丙丁戊己庚辛A-Z\\d]+[区期]$"), Pattern.compile("碧桂园"), Pattern.compile("住宅楼"), Pattern.compile("家属[区院]"), Pattern.compile("宿舍"), Pattern.compile("生活区"), Pattern.compile("公寓"));
static final Pattern INVALID_PATTERN = Pattern.compile("^([-\\\\、])+$");
@@ -42,12 +42,10 @@ public class TooltipProcessor extends AbstractAddressProcessor {
if(matchFirst(AddressParserPatterns.STREET, tooltip) != null){
address.setTooltip(tooltip);
address.setStreet(getStreet(tooltip));
} else if(containsDistrictName(tooltip)){
address.setTooltip(tooltip);
} else if(RegionData.isRegion(address.getCity(), tooltip)){
}else if(RegionData.isRegion(address.getCity(), tooltip)){
address.setSuggestDistrict(tooltip);
}else{
address.addNosie(tooltip);
address.setTooltip(tooltip);
}
}
}
@@ -15,15 +15,6 @@ public class VillageNameProcessor extends AbstractAddressProcessor {
@Override
public AbstractAddressProcessor.RepeatStatus process(Address address) {
String orgAddress = address.getAddress();
if(address.getTokens()!=null){
for(Token token:address.getTokens()){
if("district".equals(token.getKeywordTypes())){
address.setKeywords(token.getName());
address.setAddress(orgAddress.replace(token.getName(),""));
return RepeatStatus.NEXT;
}
}
}
List<String> names = tokenDistrictNames(orgAddress);
List<String> kws = new ArrayList<String>();
for(String name : names){
@@ -1,24 +1,16 @@
package com.fangjia.service.cnAnalyzer;
import java.io.IOException;
import java.io.StringReader;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fangjia.core.util.DataUtil;
import com.goojia.solr.ganalytics.IKSegmentation;
import com.goojia.solr.ganalytics.Lexeme;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.StringReader;
import java.io.UnsupportedEncodingException;
import java.util.*;
/**
* 房价网分词器
@@ -240,7 +240,7 @@ public class AddressRepoServiceImpl implements AddressRepoService {
}
private boolean isAvailablePoi(AddressMatchMetaData data){
if(data.getType() != null && data.getType().contains("·")){
if(data.getType() != null && (data.getType().contains("·") || data.getType().contains("½ÖµÀ"))){
return false;
}
return true;
@@ -0,0 +1,9 @@
package com.fangjia.service.open;
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
import com.fangjia.dal.mongo.open.model.CommunityPoi;
public interface CommunityBasePriceService {
CommunityBasePrice getById(String id);
}
@@ -0,0 +1,8 @@
package com.fangjia.service.open;
import com.fangjia.dal.mongo.open.model.CommunityPoi;
public interface CommunityPoiService {
CommunityPoi getById(String id);
}
@@ -0,0 +1,22 @@
package com.fangjia.service.open.impl;
import com.fangjia.dal.mongo.open.dao.CommunityBasePriceDao;
import com.fangjia.dal.mongo.open.dao.CommunityPoiDao;
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
import com.fangjia.dal.mongo.open.model.CommunityPoi;
import com.fangjia.service.open.CommunityBasePriceService;
import com.fangjia.service.open.CommunityPoiService;
public class CommunityBasePriceServiceImpl implements CommunityBasePriceService {
private CommunityBasePriceDao communityBasePriceDao;
@Override
public CommunityBasePrice getById(String id) {
return communityBasePriceDao.findById(id);
}
public void setCommunityBasePriceDao(CommunityBasePriceDao communityBasePriceDao) {
this.communityBasePriceDao = communityBasePriceDao;
}
}
@@ -0,0 +1,19 @@
package com.fangjia.service.open.impl;
import com.fangjia.dal.mongo.open.dao.CommunityPoiDao;
import com.fangjia.dal.mongo.open.model.CommunityPoi;
import com.fangjia.service.open.CommunityPoiService;
public class CommunityPoiServiceImpl implements CommunityPoiService {
private CommunityPoiDao communityPoiDao;
@Override
public CommunityPoi getById(String id) {
return communityPoiDao.findById(id);
}
public void setCommunityPoiDao(CommunityPoiDao communityPoiDao) {
this.communityPoiDao = communityPoiDao;
}
}
@@ -20,7 +20,7 @@
.officeMap{width:100%;height:400px;}
.main_content.partner{overflow:hidden;}
.main_content.partner a{float:left;width:180px;height:120px;background:url(../../../images/fangjia/v5/partner.png) no-repeat #f0f4f7;margin-right:24px;margin-bottom:25px;cursor:pointer;}
.main_content.partner a{float:left;width:180px;height:120px;background:url(../../../images/fangjia/v5/partner.png) no-repeat #f0f4f7;margin-right:15px;margin-bottom:25px;cursor:pointer;}
.partner a.partner11{background-position:0 0;}
.partner a.partner12{background-position:-204px 0;}