diff --git a/projects/fangjia-coresite/src/main/webapp/company/about.ftl b/projects/fangjia-coresite/src/main/webapp/company/about.ftl index 778557075..16b30ed80 100644 --- a/projects/fangjia-coresite/src/main/webapp/company/about.ftl +++ b/projects/fangjia-coresite/src/main/webapp/company/about.ftl @@ -158,37 +158,36 @@ <#-- --> - + - + - + - + - + - diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/CommunityBasePriceDao.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/CommunityBasePriceDao.java new file mode 100644 index 000000000..8275f68f8 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/CommunityBasePriceDao.java @@ -0,0 +1,9 @@ +package com.fangjia.dal.mongo.open.dao; + +import com.fangjia.dal.MongoDao; +import com.fangjia.dal.mongo.open.model.CommunityBasePrice; +import com.fangjia.dal.mongo.open.model.CommunityPoi; + +public interface CommunityBasePriceDao extends MongoDao { + +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/CommunityPoiDao.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/CommunityPoiDao.java new file mode 100644 index 000000000..97f9d48de --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/CommunityPoiDao.java @@ -0,0 +1,9 @@ +package com.fangjia.dal.mongo.open.dao; + +import com.fangjia.dal.MongoDao; +import com.fangjia.dal.mongo.model.CountDistrictStatBycity; +import com.fangjia.dal.mongo.open.model.CommunityPoi; + +public interface CommunityPoiDao extends MongoDao { + +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/impl/CommunityBasePriceDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/impl/CommunityBasePriceDaoImpl.java new file mode 100644 index 000000000..c89477b09 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/impl/CommunityBasePriceDaoImpl.java @@ -0,0 +1,10 @@ +package com.fangjia.dal.mongo.open.dao.impl; + +import com.fangjia.dal.MongoDaoImpl; +import com.fangjia.dal.mongo.open.dao.CommunityBasePriceDao; +import com.fangjia.dal.mongo.open.dao.CommunityPoiDao; +import com.fangjia.dal.mongo.open.model.CommunityBasePrice; +import com.fangjia.dal.mongo.open.model.CommunityPoi; + +public class CommunityBasePriceDaoImpl extends MongoDaoImpl implements CommunityBasePriceDao { +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/impl/CommunityPoiDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/impl/CommunityPoiDaoImpl.java new file mode 100644 index 000000000..e8a317b04 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/impl/CommunityPoiDaoImpl.java @@ -0,0 +1,10 @@ +package com.fangjia.dal.mongo.open.dao.impl; + +import com.fangjia.dal.MongoDaoImpl; +import com.fangjia.dal.mongo.open.dao.ApiVisitCountDao; +import com.fangjia.dal.mongo.open.dao.CommunityPoiDao; +import com.fangjia.dal.mongo.open.model.CommunityPoi; +import com.fangjia.dal.mongo.tool.model.ApiVisitCount; + +public class CommunityPoiDaoImpl extends MongoDaoImpl implements CommunityPoiDao { +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/model/CommunityBasePrice.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/model/CommunityBasePrice.java new file mode 100644 index 000000000..9321bd542 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/model/CommunityBasePrice.java @@ -0,0 +1,74 @@ +package com.fangjia.dal.mongo.open.model; + +import com.fangjia.dal.MongoEntity; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; + +/** + * 小区建行置信度信息 + * + * @author shenjian@fangjia.com + */ +@Entity +@Table(name = "business.community_base_price_gj") +public class CommunityBasePrice implements MongoEntity { + + private static final long serialVersionUID = -8010010773207622878L; + + private String id; + private String city; + private String region; + private String name; + private Double credit; + + @Id + @Column(name = "_id") + @Override + public String getId() { + return id; + } + + @Override + public void setId(String id) { + this.id = id; + } + + @Column(name = "city") + public String getCity() { + return city; + } + + public void setCity(String city) { + this.city = city; + } + + @Column(name = "region") + public String getRegion() { + return region; + } + + public void setRegion(String region) { + this.region = region; + } + + @Column(name = "name") + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + @Column(name = "credit") + public Double getCredit() { + return credit; + } + + public void setCredit(Double credit) { + this.credit = credit; + } +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/model/CommunityPoi.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/model/CommunityPoi.java new file mode 100644 index 000000000..8427b5aa7 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/model/CommunityPoi.java @@ -0,0 +1,135 @@ +package com.fangjia.dal.mongo.open.model; + +import com.fangjia.dal.MongoEntity; + +import javax.persistence.*; +import java.util.Date; + +/** + * 小区周边配套统计信息 + * + * @author shenjian@fangjia.com + */ +@Entity +@Table(name = "fangjia_stat.community_poi_gj") +public class CommunityPoi implements MongoEntity { + + private static final long serialVersionUID = -7663644539542318409L; + + private String id; + private String city; + private String region; + private String name; + private Integer subway; + private Integer traffic; + private Integer bank; + private Integer primary; + private Integer junior; + private Integer hospital; + /** + * 不利设施,丧葬 + */ + private Integer funeral; + + @Id + @Column(name = "_id") + @Override + public String getId() { + return id; + } + + @Override + public void setId(String id) { + this.id = id; + } + + @Column(name = "city") + public String getCity() { + return city; + } + + public void setCity(String city) { + this.city = city; + } + + @Column(name = "region") + public String getRegion() { + return region; + } + + public void setRegion(String region) { + this.region = region; + } + + @Column(name = "name") + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + @Column(name = "subway") + public Integer getSubway() { + return subway; + } + + public void setSubway(Integer subway) { + this.subway = subway; + } + + @Column(name = "traffic") + public Integer getTraffic() { + return traffic; + } + + public void setTraffic(Integer traffic) { + this.traffic = traffic; + } + + @Column(name = "bank") + public Integer getBank() { + return bank; + } + + public void setBank(Integer bank) { + this.bank = bank; + } + + @Column(name = "primary") + public Integer getPrimary() { + return primary; + } + + public void setPrimary(Integer primary) { + this.primary = primary; + } + + @Column(name = "junior") + public Integer getJunior() { + return junior; + } + + public void setJunior(Integer junior) { + this.junior = junior; + } + + @Column(name = "hospital") + public Integer getHospital() { + return hospital; + } + + public void setHospital(Integer hospital) { + this.hospital = hospital; + } + + @Column(name = "funeral") + public Integer getFuneral() { + return funeral; + } + + public void setFuneral(Integer funeral) { + this.funeral = funeral; + } +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java index bf43fa450..69108673a 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java @@ -67,12 +67,7 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i must.add(addTerms("district_name", alias)); must.add(addMatch("cat", "sell")); must.add(addRange("avg_price", minPrice, maxPrice)); - - JSONObject term = new JSONObject(); - JSONObject match = new JSONObject(); - match.put("p_date", 20210501); - term.put("match", match); - must.add(term); + must.add(addRangeGte("p_date", 20210501)); bool.put("must", must); query.put("bool", bool); @@ -168,6 +163,16 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i return term; } + private JSONObject addRangeGte(String name, Number min){ + JSONObject term = new JSONObject(); + JSONObject range = new JSONObject(); + JSONObject rangeValue = new JSONObject(); + rangeValue.put("gte", min); + range.put(name, rangeValue); + term.put("range", range); + return term; + } + private List wrapSeawater(List jsonList){ List seawaterList= new ArrayList(); @@ -210,6 +215,11 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i seawater.setFloor(source.getInt("floor")); }catch(Exception e){ seawater.setFloor(0); + } + try{ + seawater.setFloorInfo(source.getString("floor_info")); + }catch(Exception e){ + } if(source.containsKey("fitment")){ seawater.setFitment(source.getString("fitment")); diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/model/Seawater.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/model/Seawater.java index 910d805ea..1af563193 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/model/Seawater.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/model/Seawater.java @@ -48,6 +48,7 @@ public class Seawater implements SeawaterEntity { private Double area; private Integer height; private Integer floor; + private String floorInfo; private String fitment; private Integer room; private Integer hall; @@ -198,6 +199,15 @@ public class Seawater implements SeawaterEntity { this.floor = floor; } + @Column(name = "floorInfo") + public String getFloorInfo() { + return floorInfo; + } + + public void setFloorInfo(String floorInfo) { + this.floorInfo = floorInfo; + } + @Column(name = "fitment") public String getFitment() { return fitment; diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index 0b0dce39f..1874f3500 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -31,12 +31,15 @@ import javax.validation.Valid; import com.fangjia.core.util.*; import com.fangjia.dal.*; import com.fangjia.dal.mongo.model.*; +import com.fangjia.dal.mongo.open.model.CommunityBasePrice; +import com.fangjia.dal.mongo.open.model.CommunityPoi; import com.fangjia.dal.seawater.dao.SeawaterDao; import com.fangjia.dal.seawater.model.Seawater; import com.fangjia.open.dto.*; import com.fangjia.service.district.*; import com.fangjia.service.mongo.fangjia.*; import com.fangjia.service.open.*; +import com.fangjia.service.open.impl.CommunityPoiServiceImpl; import com.fangjia.service.util.DistrictScoreUtil; import net.sf.json.JSONArray; import net.sf.json.JSONObject; @@ -175,6 +178,9 @@ public class PropertyController { private SeawaterDao seawaterDao; private SeawaterService seawaterService; private SqmetreRentService sqmetreRentService; + private CommunityPoiService communityPoiService; + private CommunityBasePriceService communityBasePriceService; + private EvaluateFactorService evaluateFactorService; private FileDao fileDao = FileDao.getInstance(); private static final int DEFAULT_PAGE_SIZE = 20; @@ -390,10 +396,31 @@ public class PropertyController { put("5000", "five");// 五线 } }; + /** + * 第三方均价映射 + */ + private static final Map OTHER_PRICE_MAP = new HashMap() { + { + put("zhuge_price", "诸葛找房"); + put("anjuke_price", "安居客"); + put("fang_price", "房天下"); + put("lianjia_price", "链家"); + put("cityhouse_price", "禧泰数据"); + put("maitian_price", "麦田"); + put("ke_price", "贝壳"); + put("taofang_price", "淘房365"); + put("zhongyuan_price", "中原"); + put("tbs_price", "兔博士"); + put("qfang_price", "Q房"); + put("fangtan_price", "房探"); + put("fangduoduo_price", "房多多"); + put("wiwj_price", "我爱我家"); + } + }; /** * 利用百度API把GPS转换成Google经纬度 - * + * * @param lng * @param lat * @return @@ -415,7 +442,7 @@ public class PropertyController { /** * 利用算法模板把GPS转换成Google经纬度 - * + * * @param lng * @param lat * @return @@ -607,7 +634,7 @@ public class PropertyController { /** * 判断小区是否有经纬度 - * + * * @param district * @return */ @@ -618,7 +645,7 @@ public class PropertyController { /** * 调用匹配接口匹配小区 - * + * * @param city * @param address * @param region @@ -992,6 +1019,32 @@ public class PropertyController { } } + // 调整面积不同面积价格 + try { + DistrictStatBo districtStatBo = districtStatService.findDistrictStat(new DistrictKey(district)); + int orgAvgPrice = 0; + if (districtStatBo != null) { + if (StringUtils.isNotBlank(district.getEstateType2()) && district.getEstateType2().contains("别墅") && districtStatBo.getVillaDistrictStat() != null) { + orgAvgPrice = districtStatBo.getVillaDistrictStat().getAvgPrice(); + } else if (districtStatBo.getDistrictStat() != null) { + orgAvgPrice = districtStatBo.getDistrictStat().getAvgPrice(); + } + } + if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) { + int factorPrice = evaluation.getAvgPrice(); + int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice); + Integer minPrice = (int)(evaluation.getAvgPrice() * 0.95); + Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.05); + // 偏离度超过5%不进行调整 + if (minPrice <= avgPrice && avgPrice <= maxPrice) { + evaluation.setAvgPrice(avgPrice); + evaluation.setBigTotalPrice((long) (avgPrice * evaluation.getArea())); + } + } + } catch (Exception ex) { + + } + if (evaluation.getTotalPrice() != null && evaluation.getTotalPrice() != 0 && (cityPriceCredit == null || evaluation.getCredit() >= cityPriceCredit)) { Map districtPriceMap = new HashMap(); @@ -1162,19 +1215,52 @@ public class PropertyController { evaluationService.remove(evaluation.getId()); mav.setModel(BasicModel.buildOKModel().setMsg("暂时不支持该小区的估价").markNoData()); return mav; + } + // 调整面积不同面积价格 + try { + DistrictStatBo districtStatBo = districtStatService.findDistrictStat(new DistrictKey(district)); + int orgAvgPrice = 0; + if (districtStatBo != null) { + if (StringUtils.isNotBlank(district.getEstateType2()) && district.getEstateType2().contains("别墅") && districtStatBo.getVillaDistrictStat() != null) { + orgAvgPrice = districtStatBo.getVillaDistrictStat().getAvgPrice(); + } else if (districtStatBo.getDistrictStat() != null) { + orgAvgPrice = districtStatBo.getDistrictStat().getAvgPrice(); + } + } + if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) { + int factorPrice = evaluation.getAvgPrice(); + int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice); + Integer minPrice = (int)(evaluation.getAvgPrice() * 0.95); + Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.05); + // 偏离度超过5%不进行调整 + if (minPrice <= avgPrice && avgPrice <= maxPrice) { + evaluation.setAvgPrice(avgPrice); + evaluation.setBigTotalPrice((long) (avgPrice * evaluation.getArea())); + } + } + } catch (Exception ex) { + } Map districtPriceMap = new LinkedHashMap(); districtPriceMap.put("city", district.getCity()); districtPriceMap.put("district", district.getRegion()); districtPriceMap.put("name", district.getName()); long totalPrice = evaluation.getBigTotalPrice(); - if(CITY_CORRECTION_FACTOR.containsKey(district.getCity())){ - totalPrice = (long)(totalPrice * (1 - CITY_CORRECTION_FACTOR.get(district.getCity()))); - evaluation.setBigTotalPrice(totalPrice); - evaluation.setAvgPrice((int)(totalPrice/evaluation.getArea())); - } + // 取消系数下调 +// if(CITY_CORRECTION_FACTOR.containsKey(district.getCity())){ +// totalPrice = (long)(totalPrice * (1 - CITY_CORRECTION_FACTOR.get(district.getCity()))); +// evaluation.setBigTotalPrice(totalPrice); +// evaluation.setAvgPrice((int)(totalPrice/evaluation.getArea())); +// } districtPriceMap.put("totalPrice", evaluation.getBigTotalPrice()); districtPriceMap.put("unitPrice", evaluation.getAvgPrice()); + // 新增置信度 + CommunityBasePrice communityBasePrice = communityBasePriceService.getById(district.getId()); + if (communityBasePrice != null) { + districtPriceMap.put("credit", communityBasePrice.getCredit()); + } else { + districtPriceMap.put("credit", evaluation.getCredit()); + } List names = new ArrayList(); names.add(district.getName()); if(district.getAlias() != null){ @@ -1186,6 +1272,31 @@ public class PropertyController { } Integer minPrice = (int)(evaluation.getAvgPrice() * 0.8); Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.2); + Integer min10Price = (int)(evaluation.getAvgPrice() * 0.9); + Integer max10Price = (int)(evaluation.getAvgPrice() * 1.1); + // 新增第三方均价 + try { + double dCredit = 0.61; + Map rMap = getLastEvaluationPrice(district, evaluation, dCredit); + if (null != rMap) { + if (rMap.containsKey("other_price")) { + Map otherPriceMap = (Map) rMap.get("other_price"); + Map priceResultMap = new HashMap(); + for (String key : otherPriceMap.keySet()) { + Integer price = otherPriceMap.get(key); + // 剔除偏差大于10的 + if (min10Price <= price && max10Price >= price) { + if (OTHER_PRICE_MAP.containsKey(key)) { + priceResultMap.put(OTHER_PRICE_MAP.get(key), price); + } + } + } + districtPriceMap.put("otherPrice", priceResultMap); + } + } + } catch (Exception ex) { + + } List houses = seawaterDao.findHouses(district.getCity(), district.getRegion(), names,minPrice, maxPrice, 20); if(houses.size() > 3){ districtPriceMap.put("calcType", 1); @@ -1214,7 +1325,7 @@ public class PropertyController { }else if(district.getLng() != null && district.getLng() > 0){ double lng = DataUtil.roundDouble(district.getLng() / 10000000000.0,6); double lat = DataUtil.roundDouble(district.getLat() / 10000000000.0, 6); - List districts = districtService.findNearestDistrict(district.getEstateType2(), lng, lat, 1000 * 3 , 4, null); + List districts = districtService.findNearestDistrict(district.getEstateType2(), lng, lat, 1000 * 3 , 6, null); if(districts.size() < 2){ evaluationService.remove(evaluation.getId()); mav.setModel(BasicModel.buildOKModel().setMsg("暂时不支持该小区的估价").markNoData()); @@ -1252,9 +1363,15 @@ public class PropertyController { districtDto.append("buildYear", String.valueOf(buildYear)); } districtAnalyData = districtAnalyDataService.findByDistrictKey(new DistrictKey(d)); - if(districtAnalyData != null && districtAnalyData.getHalfYearMarkup() != null){ - districtDto.append("halfYearMarkup",DataUtil.roundDouble( districtAnalyData.getHalfYearMarkup() * 100 ,2)); + if(districtAnalyData != null){ + if (districtAnalyData.getHalfYearMarkup() != null) { + districtDto.append("halfYearMarkup", DataUtil.roundDouble(districtAnalyData.getHalfYearMarkup() * 100, 2)); + } + if (districtAnalyData.getAvgPrice() != null && minPrice <= districtAnalyData.getAvgPrice() && maxPrice >= districtAnalyData.getAvgPrice()) { + districtDto.append("avgPrice", districtAnalyData.getAvgPrice()); + } } + nearCommunityDtos.add(districtDto); } Integer buildYear = districtService.getBuildYear(district); @@ -1262,6 +1379,18 @@ public class PropertyController { modelFeaturesFto.put("buildYear", String.valueOf(buildYear)); } modelFeaturesFto.append("nearCommunity", nearCommunityDtos); + + // 模型特征新增周边数量 + CommunityPoi communityPoi = communityPoiService.getById(district.getId()); + if (communityPoi != null) { + modelFeaturesFto.put("subway", communityPoi.getSubway()); + modelFeaturesFto.put("traffic", communityPoi.getTraffic()); + modelFeaturesFto.put("bank", communityPoi.getBank()); + modelFeaturesFto.put("primary", communityPoi.getPrimary()); + modelFeaturesFto.put("hospital", communityPoi.getHospital()); + modelFeaturesFto.put("junior", communityPoi.getJunior()); + modelFeaturesFto.put("funeral", communityPoi.getFuneral()); + } districtPriceMap.put("modelFeatures", modelFeaturesFto); } mav.setModel(BasicModel.buildOKModel().setResult(districtPriceMap)); @@ -1469,7 +1598,7 @@ public class PropertyController { return mav; } } - + /** * 四川锦程 定制接口 多建筑年代 * @param model @@ -1581,7 +1710,7 @@ public class PropertyController { districtPriceMap.put("buildYear", matcher3.group()); } } - + // 根据城市等 级设置D级可信度 ApiUserAuthority apiUserAuthority = apiUserAuthorityService @@ -2094,7 +2223,7 @@ public class PropertyController { /** * 小区租金查询接口 - * + * * @param model * @return */ @@ -2573,7 +2702,7 @@ public class PropertyController { /** * 批量获取小区的均价,均价走势数据 - * + * * @param request * @return */ @@ -2715,7 +2844,7 @@ public class PropertyController { /** * 批量获取小区的配套数据,最多12个小区 - * + * * @param request * @return */ @@ -2860,7 +2989,7 @@ public class PropertyController { /** * 获取JSON参数中的num参数值 - * + * * @param jo * @return */ @@ -2885,7 +3014,7 @@ public class PropertyController { /** * 获取JSON参数中的radius值 - * + * * @param jo * @return */ @@ -3693,7 +3822,7 @@ public class PropertyController { /** * 解析出关键字中的小区名 - * + * * @param keyword * @return */ @@ -3810,7 +3939,7 @@ public class PropertyController { /** * 小区统计分析接口 - * + * * @param model * @return */ @@ -3895,7 +4024,7 @@ public class PropertyController { /** * 小区交通接口 - * + * * @param model * @return */ @@ -3953,7 +4082,7 @@ public class PropertyController { /** * 小区周边接口 - * + * * @param model * @return */ @@ -4012,7 +4141,7 @@ public class PropertyController { /** * 相同小区相似户型房源挂牌量接口
* 获取该小区的满足指定相似条件的挂牌房源量 条件说明: 1.输入面积-5<=房源面积<=输入面积+5 2.上个月发布的房源 - * + * * @author yinjihuan * @date 2016年9月1日 * @param city @@ -4052,7 +4181,7 @@ public class PropertyController { /** * 周边小区相似户型挂牌量接口
* 获取该小区的满足指定相似条件的挂牌房源量 条件说明: 1.输入面积-5<=房源面积<=输入面积+5 2.上个月发布的房源 - * + * * @author yinjihuan * @date 2016年9月1日 * @param city @@ -4103,7 +4232,7 @@ public class PropertyController { /** * 小区周边一公里均价 - * + * * @author wenyan * @param model * @return @@ -4126,7 +4255,7 @@ public class PropertyController { /** * 相同小区相似户型租金范围查询接口 - * + * * @param model * @return */ @@ -4169,7 +4298,7 @@ public class PropertyController { /** * 周边小区相似户型租金范围查询接口 - * + * * @param model * @return */ @@ -4232,7 +4361,7 @@ public class PropertyController { /** * 室号查询接口 - * + * * @param model * @return */ @@ -4268,7 +4397,7 @@ public class PropertyController { /** * 楼栋楼层信息查询接口 - * + * * @param model * @return */ @@ -4293,7 +4422,7 @@ public class PropertyController { /** * 楼栋查询接口 - * + * * @param model * @return */ @@ -4338,7 +4467,7 @@ public class PropertyController { /** * 楼栋查询接口 - * + * * @param model * @return */ @@ -4791,7 +4920,7 @@ public class PropertyController { /** * 房产交易记录查询 - * + * * @param model * @return */ @@ -4851,7 +4980,7 @@ public class PropertyController { /** * 房产拍卖记录 - * + * * @param model * @return */ @@ -4912,7 +5041,7 @@ public class PropertyController { /** * 房产近期评估次数 - * + * * @param model * @return */ @@ -5627,4 +5756,16 @@ public class PropertyController { public void setRentDealPriceService(RentDealPriceService rentDealPriceService) { this.rentDealPriceService = rentDealPriceService; } + + public void setCommunityPoiService(CommunityPoiService communityPoiService) { + this.communityPoiService = communityPoiService; + } + + public void setEvaluateFactorService(EvaluateFactorService evaluateFactorService) { + this.evaluateFactorService = evaluateFactorService; + } + + public void setCommunityBasePriceService(CommunityBasePriceService communityBasePriceService) { + this.communityBasePriceService = communityBasePriceService; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/filters/AnalyticsFilter.java b/projects/fangjia-open/src/main/java/com/fangjia/open/filters/AnalyticsFilter.java index e16664d16..37805ef21 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/filters/AnalyticsFilter.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/filters/AnalyticsFilter.java @@ -75,8 +75,8 @@ public class AnalyticsFilter implements Filter { if ("1".equals(request.getParameter("__refresh_cache"))) { threadLocalData.put(ThreadLocalKey.DisableCache, "1"); } - createUID(request, response); - createSID(request, response); + //createUID(request, response); + //createSID(request, response); response.addHeader("Access-Control-Allow-Origin", "*"); response.addHeader("Access-Control-Allow-Methods", "*"); chain.doFilter(request, response); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index c2f8fe431..6cd17549e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -27,7 +27,7 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataCommunityBatch extends BaseBatch{ - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) static main(args) { runBatch(new BigdataCommunityBatch(), args); @@ -187,11 +187,22 @@ class BigdataCommunityBatch extends BaseBatch{ }else{ def jsonObj = null if(dist.address){ - jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name) +// jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name) + jsonObj = OpenApiClient.standard1Action(dist.city, dist.address) } - if(jsonObj && jsonObj.searchAddress){ - ret << jsonObj.searchAddress.street - ret << jsonObj.searchAddress.streetNumber + if(jsonObj && jsonObj.street){ + if(jsonObj.street){ + ret << jsonObj.street + }else{ + ret << "" + } + + if(jsonObj.streetNumber){ + ret << jsonObj.streetNumber + }else{ + ret << "" + } + }else{ ret << "" ret << "" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 4978bd962..0bc461cc5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220318, 20220419); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220418, 20220519); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); //测试通过匹配接口配房源效果 // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118) @@ -97,7 +97,7 @@ class BigdataForsaleBatch extends BaseBatch{ totalCount.addAndGet(1) def commId = rMap.get("_id").toString() - def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias) + def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias, rMap.estate_type2) //def ret = findHousesByDistId(city, commId, "sell", startDate,endDate) // if(ret && ret.size()<100){//导历史房源需要,以后不需要 // ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias) @@ -408,7 +408,7 @@ class BigdataForsaleBatch extends BaseBatch{ } - def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){ + def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){ QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false); QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); @@ -449,6 +449,13 @@ class BigdataForsaleBatch extends BaseBatch{ def map = houses[k]; DBObject seawater = new BasicDBObject(map) + if(estate_type2 && estate_type2=="普通住宅"){ + if(seawater.estate_type&& seawater.estate_type.contains("商住")){ + println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type + continue; + } + } + if(!seawater.total_price){ continue; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy index 3fffa870c..74474aff6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy @@ -57,6 +57,12 @@ class CcbFtpUtils { ftpUpload(dataCodeFileName)//上传加密码文件到FTP } + static def listFileNames(month){ + //上传到ftp + sftpChannel = reConnect(); + Vector files = sftp.listFiles(ftpPath+ month+"/", sftpChannel); + return files + } static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index d7840e5bb..c8697c956 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -15,8 +15,8 @@ class DealPriceDataProcess { - def sqlInstance2 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", - "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") +// def sqlInstance2 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", +// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") @@ -373,16 +373,17 @@ class DealPriceDataProcess { def citys = CcbCommonUtils.citys def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def publishTime = '2021-10-01' - def month = 202203 + def month = 202204 def rows = [] for(def city : citys) { - def rets = findPeGteThirdParty(city, 202204) + def rets = findPeGteThirdParty(city, 202205) def total = 0 + println city +" "+rets.size() for(def m : rets){ def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}" sqlInstance236.executeUpdate(sql) } - println city +" "+rets.size() + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 060914122..af8723b42 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -38,8 +38,8 @@ class ExportOracleData { exportCommunityBuileType() exportThirdPartyPrice2(lastMonth) - exportBigdataForsale("2022-03-19") - exportBigdataDeal("2022-03-01") + exportBigdataForsale("2022-04-19") + exportBigdataDeal("2022-04-01") //计算结果相关 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy index edc128cf1..7258934fa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy @@ -3,8 +3,10 @@ package com.fangjia.script.ccbevaluation import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.util.DateUtils +import com.jcraft.jsch.ChannelSftp import groovy.sql.Sql import com.fangjia.script.ccbevaluation.ronghe.SSHHelper +import org.apache.commons.codec.digest.DigestUtils class ExportOracleEvaluationData { def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", @@ -33,6 +35,8 @@ class ExportOracleEvaluationData { exportCommCaseDetailGj(curMonth) def curDate = curMonth.toString()+"28" +// 生成MD5文件 + exportMd5() SSHHelper jschCommand = new SSHHelper(); System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}")); // ok文件 @@ -413,4 +417,23 @@ class ExportOracleEvaluationData { CcbFtpUtils.ftpUpload(null, dataCodeFileName, curDate) } + def exportMd5(){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + Vector files = CcbFtpUtils.listFileNames(curDate) + Iterator fileVec = files.iterator() + def fileRows = [] + while(fileVec.hasNext()){ + ChannelSftp.LsEntry file = fileVec.next() + if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){ + fileRows << file.getFilename() +":"+ DigestUtils.md5Hex(file.getFilename()) + } + } + def fileName = "/ccb/md5.txt" + if(CommonDatautils.isWindow()){ + fileName = "D:\\房价网\\md5.txt" + } + FileUtils.createTxtFile(fileRows, fileName, "utf-8", false); + CcbFtpUtils.ftpUpload(null, fileName, curDate) + println "md5.txt 上传Ftp完成......" + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy index 42ad78607..b0476927d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy @@ -1,21 +1,29 @@ package com.fangjia.script.ccbevaluation +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.fangjia.core.RegionData import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import groovy.sql.Sql +import org.bson.types.ObjectId class LoadGuide { public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017) + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + static main(args) { def task = new LoadGuide() task.run() } def run(){ + addPersonGuided("D:\\数据\\建行\\人工指导价更新.xlsx") // match("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx") // saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞指导价218.xlsx") @@ -26,9 +34,51 @@ class LoadGuide { // savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx") // savePrice("D:\\数据\\建行\\房产估值\\指导价\\无锡100 广州96 二手住房成交参考价格表.xlsx") // savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx") - savePrice("D:\\数据\\建行\\房产估值\\指导价\\三亚指导价30.xlsx") +// savePrice("D:\\数据\\建行\\房产估值\\指导价\\三亚指导价30.xlsx") } + + def addPersonGuided(filename){ + ExcelUtils.modify(filename, { d -> + def city = d['B'] + if(city){ + def cityCodeDatas = CcbCommonUtils.findCityCodeValue() + + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + createSql( d, cityCodeDatas, distCodeMap) + } + + }) + } + + def createSql(d, cityCodeDatas, distCodeMap){ + def id = new ObjectId() + def commid = d['A'] + def cityId = cityCodeDatas.get(d['B']) + def sampleRegion = d['C'].replace("街道","").replace("镇","") + def distRegion = distCodeMap.get(sampleRegion) + def cityName = d['B'] + def distName = d['C'] + def commName = d['D'] + def price = d['E'].toDouble().toInteger() + + def dist = CommonDatautils.findDistrict(cityName, sampleRegion, commName, "district") + if(dist){ + def createtime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + def dealDate = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd") + + + def sql = "insert into guided_price_gj(ID,COMMID,COMMNAME,CITYID,CITYNAME,DISTID,DISTNAME,DEALDATE,AVG_PRICE,GUIDING_PERSON,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES " + + "('${id.toString()}','${dist._id.toString()}', '${commName}','${cityId}','${cityName}','${distRegion.district_code}','${distName}',to_date('${dealDate}','yyyy-mm-dd'),${price},'房价网','',to_date('${createtime}','yyyy-mm-dd hh24:mi:ss'),to_date('${createtime}','yyyy-mm-dd hh24:mi:ss'),'FJW')" + println sql + sqlInstance.executeUpdate(sql) + }else{ + println " 没查到 "+commName + } + + + } + def match(def filename){ def city = "西安" ExcelUtils.modify(filename,{d-> diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrict2Mongo85.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrict2Mongo85.groovy new file mode 100644 index 000000000..93f18dfce --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrict2Mongo85.groovy @@ -0,0 +1,67 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.core.RegionData +import com.fangjia.core.util.DistrictCodeUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import groovy.sql.Sql +import org.apache.lucene.queryparser.classic.QueryParser +import org.apache.poi.ss.util.RegionUtil +import org.bson.types.ObjectId +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.sql.ResultSet +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * oracle中小区统计数据同步至85中 + */ +class OracleDistrict2Mongo85 { + + static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + static final DBCollection communityPoiConn = MongoUtils.getCollectionByName("fangjia_stat", "community_poi_gj", "192.168.0.85", 28017) + + static main(args) { + // 只写入重庆市数据 + def sql = "select cityid, distname, commid, commname, subway, traffic, bank, primary, junior, hospital, funeral from COMMUNITY_POI_GJ" + def count = 0 + sqlInstance.eachRow(sql, { r -> + def city = DistrictCodeUtils.getCityName(r['cityid']) + def region = RegionData.formatRegion(city, r['distname']) + DBObject dbObject = new BasicDBObject("_id", r['commid']) + DBObject updateObject = new BasicDBObject("city", city).append("region", region) + .append("name", r['commname']) + .append('subway', Integer.parseInt(r['subway'].toString())) + .append("traffic", Integer.parseInt(r['traffic'].toString())) + .append("bank", Integer.parseInt(r['bank'].toString())) + .append("primary", Integer.parseInt(r['primary'].toString())) + .append("junior", Integer.parseInt(r['junior'].toString())) + .append("hospital", Integer.parseInt(r['hospital'].toString())) + .append("funeral", Integer.parseInt(r['funeral'].toString())) + .append("c_date", new Date()); + communityPoiConn.update(dbObject, updateObject, true, false) + count++; + if (count % 500 == 0) { + println "已写入" + count + "条小区数据" + } + }); + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrictBasePrice2Mongo85.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrictBasePrice2Mongo85.groovy new file mode 100644 index 000000000..a824a3d7e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrictBasePrice2Mongo85.groovy @@ -0,0 +1,52 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.core.RegionData +import com.fangjia.core.util.DistrictCodeUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.DataUtil +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import groovy.sql.Sql + +/** + * oracle中小区置信度数据同步至85中 + */ +class OracleDistrictBasePrice2Mongo85 { + + static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + static final DBCollection communityBasePriceConn = MongoUtils.getCollectionByName("business", "community_base_price_gj", "192.168.0.85", 28017) + + static main(args) { + def sql = "SELECT cityname, distname, commid, commname, confidence FROM community_base_price_gj WHERE YEAR_MONTH='202205'" + def count = 0 + sqlInstance.eachRow(sql, { r -> + def city = r['cityname'] + def region = RegionData.formatRegion(city, r['distname']) + DBObject dbObject = new BasicDBObject("_id", r['commid']) + if (r['confidence'] != null) { + def credit = new BigDecimal(r['confidence']).setScale(2, BigDecimal.ROUND_HALF_UP).toDouble() + DBObject updateObject = new BasicDBObject("city", city).append("region", region) + .append("name", r['commname']) + .append("credit", credit) + .append("c_date", new Date()); + communityBasePriceConn.update(dbObject, updateObject, true, false) + count++; + } else { + print "置信度为空:" + r.toString() + } + if (count % 500 == 0) { + println "已写入" + count + "条小区数据" + } + }); + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index e262bd7c5..d1cc4209b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202203//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202201 - def endDate = 202203 + def sDate= 202204//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202202 + def endDate = 202204 def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index 3ed13ebfc..3b3e6a7c5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -40,8 +40,8 @@ class TradeStatLog { static main(args) { def task = new TradeStatLog() -// task.run()//统计最近3个月抓取的成交数据量 - task.exportTrade()//导出未入库成交 + task.run()//统计最近3个月抓取的成交数据量 +// task.exportTrade()//导出未入库成交 // task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) // task.averageStat() @@ -75,13 +75,13 @@ class TradeStatLog { def run(){ def citys = CcbCommonUtils.citys; def rets = [] - rets << ["城市","1月总量","2月总量","3月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] + rets << ["城市","2月总量","3月总量","4月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"] //def startDate = "2021-05-01" - def startDate = "2022-01-01" - def date = "2022-02-01" - def endDate = "2022-03-01" - def newDate = "2022-04-01" + def startDate = "2022-02-01" + def date = "2022-03-01" + def endDate = "2022-04-01" + def newDate = "2022-05-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) @@ -131,7 +131,7 @@ class TradeStatLog { } - ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-03成交统计.xlsx", rets, "数据") + ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-04成交统计.xlsx", rets, "数据") println "表格生成完成" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDataStat.groovy new file mode 100644 index 000000000..1aa5a391f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDataStat.groovy @@ -0,0 +1,155 @@ +package com.fangjia.script.ccbevaluation.data + +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import groovy.sql.Sql + +class CcbDataStat { + + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + static main(args) { + def task = new CcbDataStat() +// task.run() + task.tradeRun() + } + + def tradeRun(){ + def ccbCitys = CcbCommonUtils.citys; + def startDate = DateUtils.parseFromTightByPattern("2021-01-01","yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd") + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\成交案例统计_0531.xlsx", { d -> + if(d['A']){ + if(ccbCitys.contains(d['A'])){ + d['D'] = findOracleTotalCount(d['A']) + }else{ + def totalcount = findAllTradeCount(d['A'], startDate, endDate) + def repeatCount = findRepeatCount(d['A'], startDate, endDate) + def matchTotalCount = findMatchTradeCount(d['A'], startDate, endDate) + def matchRepeatTotalCount = findMatchRepeatTradeCount(d['A'], startDate, endDate) + d['B'] = totalcount + d['C'] = repeatCount + d['D'] = matchTotalCount + d['E'] = matchRepeatTotalCount + } + + } + }) + } + + + + def findRepeatCount(city, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + + def dbcursor = DEAL_PRICE_COLL.find(q) + def set = new HashSet() + for(def d : dbcursor){ + def key = d.region+"##"+d.district_name+"##"+d.avg_price+"##"+d.area+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyyMMdd") + if(!set.contains(key)){ + set << key + } + } + return set.size() + } + + def findMatchRepeatTradeCount(city, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + q.put("match_id", new BasicDBObject('$exists', true)) + def dbcursor = DEAL_PRICE_COLL.find(q) + def set = new HashSet() + for(def d : dbcursor){ + def key = d.match_id+"#"+d.avg_price+"##"+d.area+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyyMMdd") + if(!set.contains(key)){ + set << key + } + } + return set.size() + } + + + def findMatchTradeCount(city, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + q.put("match_id", new BasicDBObject('$exists', true)) + def count = DEAL_PRICE_COLL.count(q) + return count + } + + + public static def findAllTradeCount(city, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) +// q.put("match_id", new BasicDBObject('$exists', false)) + def total = DEAL_PRICE_COLL.count(q) + return total + } + + def findOracleTotalCount(city){ + def sql = "select count(*) as count from bigdata_deal_gj where cityname='${city}' and dealdate>=to_date('2021-01-01','yyyy-MM-dd') and dealdate + count = r.count + }) + return count + } + + def run(){ + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\建行成交\\92个城市字段覆盖率情况.xlsx",{ d-> + if(d['A']) { + def citycode = cityCodeDataMap.get(d['A']) + def total = findTotalCount(citycode) + def f_count = findDistFinishDate(citycode) + def b_count = findBuildTypeCount(citycode) + def f_pre = DataHandleUtils.getDoubleRound(2, f_count*100/total) + def b_pre = DataHandleUtils.getDoubleRound(2, b_count*100/total) + d['C'] = f_pre + d['D'] = b_pre + d['E'] = total + println d['A'] +" "+ total +" "+f_count+" "+b_count + } + }) + } + + def findTotalCount(citycode){ + def sql = "select count(*) as count from bigdata_community_gj where cityid='${citycode}' and status=0";//where cityId in (350200,150100) + def count = 0 + sqlInstance.eachRow(sql, { r -> + count = r.count + }) + return count + } + + def findDistFinishDate(citycode){ + def sql = "select count(*) as count from bigdata_community_gj where cityid='${citycode}' and status=0 and BUILDYEAR>0";//where cityId in (350200,150100) + def count = 0 + sqlInstance.eachRow(sql, { r -> + count = r.count + }) + return count + } + + def findBuildTypeCount(citycode){ + def sql = "select count(*) as count from (select commid, count(commid) as count from bigdata_commbuildtype_gj where cityid='${citycode}' and BUILDTYPE!='0199' group by commid)";//where cityId in (350200,150100) + def count = 0 + sqlInstance.eachRow(sql, { r -> + count = r.count + }) + return count + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistNameMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistNameMatch.groovy new file mode 100644 index 000000000..32725d092 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistNameMatch.groovy @@ -0,0 +1,117 @@ +package com.fangjia.script.ccbevaluation.data + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient +import groovy.sql.Sql + +class CcbDistNameMatch { + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + static main(args) { + def task = new CcbDistNameMatch(); +// task.run() + task.run1() + } + + def run1(){ + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\新增映射别名.xlsx", { d -> + if (d['A']) { + def region = d['B'] + def newRegion = RegionData.formatRegion(d['A'], region) + if (newRegion) { + if(d['D'].contains("、")){ + println d['D'] + } + def arrs = d['D'].split("、") + def list = [] + def flags = [] + for(def s : arrs){ + def dist = CommonDatautils.findDistrict(d['A'], newRegion, s, "district") + if(dist){ + list << dist.name + flags << "完全一致" + }else{ + def jsonObject = OpenApiClient.addressMatch(d['A'], s, "property", false, newRegion) + if(jsonObject != null && jsonObject.precise!=-1) { + if(jsonObject.get("district") == newRegion){ + list << jsonObject.get("name") + flags << "匹配接口" + } + } + } + + } + if(list){ + list = list.unique() + d['E'] = list.join(",") + d['F'] = flags.join(",") + } + } + } + }) + } + + def run(){ + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\不同小区.xlsx", { d -> + def regionCode = null + if (d['A']) { + def region = d['B'] + def newRegion = RegionData.formatRegion(d['A'], region) + if (newRegion) { + def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") + if(dist){ + d['D'] = dist.name + def count = findDistrictByOracle(dist._id.toString()) + if(count && count>0){ + d['F'] = 1 + }else{ + d['H'] = dist.estate_type2 + d['J'] = dist.name_cat + } + }else{ + def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['C'], "district") + if (rets) { + if(rets.size()>1){ + d['E'] = "别名-多个" + }else{ + dist = rets[0] + d['E'] = d['C'] +"--"+dist.name + def count = findAliasByOracle(dist._id.toString()) + if(count && count>0) { + d['G'] = 1 + }else{ + d['H'] = dist.estate_type2 + d['J'] = dist.name_cat + } + } + } + } + } + } + }); + } + + def findDistrictByOracle(commid){ + def sql = "select count(*) as count from bigdata_community_gj where commid='${commid}' and status=0"; + def count = 0 + + sqlInstance.eachRow(sql,{r-> + count = r.count + }); + return count + } + + def findAliasByOracle(commid){ + def sql = "select count(*) as count from address_alias_fjw where address_id='${commid}'"; + def count = 0 + + sqlInstance.eachRow(sql,{r-> + count = r.count + }); + return count + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy index 2db54e10a..98d755434 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy @@ -7,6 +7,7 @@ import com.fangjia.core.util.GeoUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.gj.district.NameProcess import com.fangjia.script.linxsh.util.AddressUtil import com.fangjia.script.util.ABuYunUtil import com.fangjia.script.util.CommonUtils @@ -26,9 +27,9 @@ class CcbDistrictToDB { // task.run() // task.run1() // task.meituanRun() -// task.meituanSqrentRun() + task.meituanSqrentRun() // task.updateDistrict() - task.updateDongguangRegion() +// task.updateDongguangRegion() } def updateDongguangRegion(){ @@ -80,13 +81,17 @@ class CcbDistrictToDB { } def meituanSqrentRun(){ - ExcelUtils.modify("D:\\数据\\美团\\美团需入库小区合并后.xlsx", { d -> + def index = 0 + ExcelUtils.modify("D:\\数据\\美团\\美团待入库.xlsx", { d -> + index++ + println index+" "+d['H'] def city = d['F'] def region = d['G'] def newRegion = RegionData.formatRegion(city, region) if(newRegion) { - def name = d['H'] + def name = NameProcess.nameStandard(city, newRegion, d['H']) name = nameFileter(name) + if(name){ def estateType2 = "普通住宅" if(name.contains("别墅")){ @@ -103,7 +108,6 @@ class CcbDistrictToDB { } } } - def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1)); CommonUtils.recalculationSqRent(city, region, name, estateType2, lastMonthDay,false,0) } @@ -115,97 +119,100 @@ class CcbDistrictToDB { def index = 0//美团需入库小区合并后.xlsx def keys = new CopyOnWriteArrayList() - ExcelUtils.multiThreadingModify("D:\\数据\\美团\\美团需入库小区合并后.xlsx", { d -> + ExcelUtils.multiThreadingModify("D:\\数据\\美团\\美团待入库.xlsx", { d -> + index++ + println index+" "+d['H'] def city = d['F'] def region = d['G'] def newRegion = RegionData.formatRegion(city, region) - if(newRegion){ - def name = d['H'] + if(newRegion) { + def name = NameProcess.nameStandard(city, newRegion, d['H']) name = nameFileter(name) + if (name) { def formatAddress = null - if(d['E']){ - formatAddress = AddressUtil.formatAddress(city, d['E']) + if (d['I']) { + formatAddress = AddressUtil.formatAddress(city, d['I']) } - if(!formatAddress || (formatAddress && !formatAddress.contains("号") && !formatAddress.contains("交叉"))){ - def rets = OpenApiUtils.getGaoDeSearchLocation(city, region, name) - def map = findStandardObject2(city,region, name, d['E'], rets); - if(map && map.address){ - formatAddress = AddressUtil.formatAddress(city, map.address) - if(!formatAddress){ - if(map.address.contains("交叉口")){ - formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉口")+3) - }else if(map.address.contains("交叉")){ - formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉")+2) - } - } - } - } +// if(!formatAddress || (formatAddress && !formatAddress.contains("号") && !formatAddress.contains("交叉"))){ +// def rets = OpenApiUtils.getGaoDeSearchLocation(city, region, name) +// def map = findStandardObject2(city,region, name, d['E'], rets); +// if(map && map.address){ +// formatAddress = AddressUtil.formatAddress(city, map.address) +// if(!formatAddress){ +// if(map.address.contains("交叉口")){ +// formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉口")+3) +// }else if(map.address.contains("交叉")){ +// formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉")+2) +// } +// } +// } +// } Long glat = null Long glng = null Long blat = null Long blng = null - if(d['K']){ - glat = d['L'].toString().toDouble() * 10000000000L - glng = d['K'].toString().toDouble() * 10000000000L - def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3) + if (d['L']) { + glat = d['M'].toString().toDouble() * 10000000000L + glng = d['L'].toString().toDouble() * 10000000000L + def baidu = BaiDuApiUtils.geoconv(d['L'].toString().toDouble(), d['M'].toString().toDouble(), 3) blng = baidu.x * 10000000000L blat = baidu.y * 10000000000L } def estateType2 = "普通住宅" - if(name.contains("别墅")){ + if (name.contains("别墅")) { estateType2 = "别墅" - }else if(name.contains("商住")){ + } else if (name.contains("商住")) { estateType2 = "商住" - }else{ - def type = d['I'] - if(type){ - if(type.contains("别墅")){ + } else { + def type = d['J'] + if (type) { + if (type.contains("别墅")) { estateType2 = "别墅" - }else if(type.contains("商住")){ + } else if (type.contains("商住")) { estateType2 = "商住" } } } - def key = city+"##"+newRegion+"##"+name - if(name && !keys.contains(key)){ + def key = city + "##" + newRegion + "##" + name + if (name && !keys.contains(key)) { def dist = CommonDatautils.findDistrict(city, newRegion, name, "district") - if(!dist){ + if (!dist) { keys << key def data = new BasicDBObject(); - data.put("name_cat",0) - data.put("status",0) - data.put("cat","district") + data.put("name_cat", 12) + data.put("status", 0) + data.put("cat", "district") data.put("c_date", new Date()) data.put("m_date", new Date()) data.put("city", city) - data.put("region",newRegion) + data.put("region", newRegion) data.put("name", name) - data.put("estate_type2",estateType2) + data.put("estate_type2", estateType2) data.put("address", formatAddress) - if(glat){ - data.put("lat",glat) - data.put("lng",glng) - data.put("lat2",blat) - data.put("lng2",blng) + if (glat) { + data.put("lat", glat) + data.put("lng", glng) + data.put("lat2", blat) + data.put("lng2", blng) } def markMap = [:] markMap.put("data_source", "机器入库") - data.put("mark_map",markMap) + data.put("mark_map", markMap) index++ DbUtils.seaweed.save(data); - println index+" "+data + println index + " " + data def stop = false - while (!stop){ - if(data._id.toString()){ + while (!stop) { + if (data._id.toString()) { addPrice(data) stop = true; } @@ -213,19 +220,17 @@ class CcbDistrictToDB { } } + } }else{ println city +" "+region +" 区域无法标准化" } },0); - - - } def addPrice(dist){ - def startDate = DateUtils.parseFromTightYYYYMMDD("2022-03-01") - def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-25") + def startDate = DateUtils.parseFromTightYYYYMMDD("2022-05-01") + def endDate = DateUtils.parseFromTightYYYYMMDD("2022-05-24") def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate) for(def weekend : weekends){ @@ -435,9 +440,12 @@ class CcbDistrictToDB { } def nameFileter(def name){ - name = name.replace("·","").replace(".","").replace("|","") - name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期") - name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区") + if(name){ + name = name.replace("·","").replace(".","").replace("|","") + name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期") + name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区") + } + return name; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/DistrictMarge.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/DistrictMarge.groovy index a1e095a7f..afa2233b6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/DistrictMarge.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/DistrictMarge.groovy @@ -1,10 +1,12 @@ package com.fangjia.script.ccbevaluation.data +import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.linxsh.util.AddressUtil import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject +import com.mongodb.DBObject import org.apache.poi.ss.util.RegionUtil import org.bson.types.ObjectId import com.fangjia.core.util.BaiDuApiUtils @@ -13,7 +15,82 @@ class DistrictMarge { static main(args) { def task = new DistrictMarge() //task.run() - task.repeatAddress() +// task.repeatAddress() +// task.removeDist() + task.updateAddress() +// task.addMappingAlias() + } + def updateAddress(){ + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\更新小区地址.xlsx", { d -> + if (d['A']) { + def region = d['B'].trim() + def newRegion = RegionData.formatRegion(d['A'], region) + if (newRegion) { + def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") + if (dist) { + DBObject udata = new BasicDBObject() + udata.put("address", d['D']) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set', udata)) + } + }else{ + println d['A'] + " 格式化失败 "+d['B'] + } + } + }); + } + + def removeDist(){ + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\删除小区.xlsx", { d -> + if (d['A']) { + def region = d['B'] + def newRegion = RegionData.formatRegion(d['A'], region) + if (newRegion) { + def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") + if (dist) { + DBObject udata = new BasicDBObject() + udata.put("status", 1) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set', udata)) + } + }else{ + println d['A'] + " 格式化失败 "+d['B'] + } + } + }) + } + + def addMappingAlias(){ + def index = 0 + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\新增映射别名.xlsx", { d -> + index ++ + if (d['A']) { + def region = d['B'].trim() + def newRegion = RegionData.formatRegion(d['A'], region) + if (newRegion) { + def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") + if (dist) { + def mapping_alias = dist.mapping_alias + if(!dist.mapping_alias){ + mapping_alias = [] + } + def alias = d['D'] + def arrs = alias.split("、") + for(def s : arrs){ + if(!mapping_alias.contains(s)){ + mapping_alias << s + } + } + DBObject udata = new BasicDBObject() + udata.put("mapping_alias", mapping_alias) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set', udata)) + } + }else{ + println d['A'] + " 格式化失败 "+d['B'] + } + } + }) } def repeatAddress(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy index 2f5d02c2f..a62cf530f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy @@ -36,10 +36,37 @@ class MatchMapData { // task.run3() // task.exportHouse() // task.addDistrictLevel() - task.run4() +// task.run4() + task.run5() } + def run5(){ + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\数据核实.xlsx", { d -> + if (d['A']) { + def region = d['B'] + def newRegion = RegionData.formatRegion(d['A'], region) + if (newRegion) { + def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") + if(dist){ + if(dist.mapping_alias){ + d['D'] = dist.mapping_alias.join(",") + } + d['E'] = dist.address + } + + def dist1 = CommonDatautils.findDistrict(d['A'], newRegion, d['G'], "district") + if(dist1){ + if(dist1.mapping_alias){ + d['H'] = dist1.mapping_alias.join(",") + } + d['I'] = dist1.address + } + } + } + }); + } + def run4() { ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\核实小区是否存在.xlsx", { d -> def regionCode = null diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/OracleHouse.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/OracleHouse.groovy new file mode 100644 index 000000000..ebb9c165b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/OracleHouse.groovy @@ -0,0 +1,108 @@ +package com.fangjia.script.ccbevaluation.data + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.ElasticUtils +import com.mongodb.BasicDBObject +import groovy.sql.Sql +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.sql.ResultSet +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class OracleHouse { + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + + public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" + public static EsDataSource dataSource = new EsDataSource() + static { + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + connectionProperties.put("user", "developer") + connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") + dataSource.setProperties(connectionProperties) + } + + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + + static main(args) { + def task = new OracleHouse() + task.run() + } + + def run(){ + def rets = findAllHouse("2022-04-19") + def totalCount = new AtomicLong() + println rets.size() + final CountDownLatch latch = new CountDownLatch(rets.size()) + for(def i=0;i + index = index + 1 + println index + def data = [] + data << r.FORSALEID + data << r.COMMID + allRows << data + }); + return allRows + } + + def removeHouse(houseid){ + def sql = "delete from bigdata_forsale_gj where FORSALEID='${houseid}' "; + sqlInstance.executeUpdate(sql) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy index bae011454..9ac21fba3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy @@ -2,39 +2,70 @@ package com.fangjia.script.chongqing import com.fangjia.core.RegionData import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.core.util.DateUtil +import com.fangjia.httpclient.UserAgents import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.gj.paimai.DataUtil import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.util.ABuYunUtil import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.JacksonUtil import com.fangjia.script.util.OpenApiClient +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.google.gson.JsonObject import com.mongodb.BasicDBObject +import com.mongodb.DBObject import groovy.sql.Sql +import net.sf.json.JSONObject +import org.apache.commons.lang.StringUtils +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders import java.util.concurrent.CopyOnWriteArrayList +import java.util.zip.GZIPInputStream class MatchEvaluateRegionTest { + static def targetUrl = "http://192.168.0.12:9200/district_comp_potential/" def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + def path = "D:\\数据\\美团\\10城小区明细-V10.xlsx" + def total = 1500 + static main(args) { def task = new MatchEvaluateRegionTest() - task.run()//小区匹配 - task.run1()//地图匹配 +// task.run()//小区匹配 +// task.run1()//地图匹配 // task.addDistrict();//新增人工处理小区,原文件匹配不上的去除 - task.addPrice() - task.addAuthor() +// task.addPrice() +// task.addArroundPrice() +// task.addArroundPriceByLat() +// task.addRentRate() +// task.addAuthor() +// task.addLevel() +// task.addSorce() +// task.addArroundScore() + + task.pkPrice() + } def run(){ - def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx" + + def index = 0 ExcelUtils.multiThreadingModify(path, { row -> + index ++ if(!row['F']){ - def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], "property", false, null, null) + println index+" "+row['E'] + def jsonObject = OpenApiClient.addressMatch(row['A'], row['C'], "property", false,row['B'],null) if(jsonObject != null && jsonObject.precise!=-1) { row['F'] = jsonObject.get("id") row['G'] = jsonObject.get("city") @@ -43,18 +74,17 @@ class MatchEvaluateRegionTest { row['J'] = jsonObject.get("address") row['K'] = jsonObject.get("propertyType") row['L'] = jsonObject.get("map") -// row['K'] = jsonObject.get("lng") -// row['L'] = jsonObject.get("lat") + row['M'] = jsonObject.get("lng") + row['N'] = jsonObject.get("lat") } } }); } def run1(){ - def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx" ExcelUtils.multiThreadingModify(path, { row -> - if(!row['F']){ - def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], null, true, null, null) + if(!row['F'] && row['A']){ + def jsonObject = OpenApiClient.addressMatch(row['A'], row['E'], null, true, row['B'],row['C'].toString()) if(jsonObject != null && jsonObject.precise!=-1) { row['F'] = jsonObject.get("id") row['G'] = jsonObject.get("city") @@ -63,52 +93,104 @@ class MatchEvaluateRegionTest { row['J'] = jsonObject.get("address") row['K'] = jsonObject.get("propertyType") row['L'] = jsonObject.get("map") -// row['K'] = jsonObject.get("lng") -// row['L'] = jsonObject.get("lat") + row['M'] = jsonObject.get("lng") + row['N'] = jsonObject.get("lat") } } }); } + def pkPrice(){ + ExcelUtils.multiThreadingModify(path, { row -> + if(row['R'] && row['R'].toString().toDouble().toInteger()==1){ + def obj = CommonDatautils.queryDistrictPrice(row['G'], row['H'], row['I'], DateUtils.parseFromTightByPattern("2022-05-22","yyyy-MM-dd")) + if(!obj){ + row['R'] = '' + row['S'] = "无最新价" + } + } + }); + + } + def addPrice(){ - def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx" ExcelUtils.multiThreadingModify(path, { row -> if(!row['L']){ def priceObj = CommonDatautils.findDistrictPrice(row['G'], row['H'], row['I']) if(priceObj){ - row['M'] = priceObj.avg_price - row['N']= (int)(priceObj.avg_price * row['E']) + row['O'] = priceObj.avg_price + // row['N']= (int)(priceObj.avg_price * row['E']) } - def avg_price = findAvgPrice(row['M']) - if(row['M'] && avg_price){ - def weekendPrice = row['M'].toString().toDouble() - def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price) - row['O'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15) + def sqmetreRentPrice = CommonDatautils.findNewSqmetreRent(row['G'], row['H'], row['I']) + if(sqmetreRentPrice){ + row['P'] = sqmetreRentPrice } + +// def avg_price = findAvgPrice(row['F']) +// if(row['M'] && avg_price){ +// def weekendPrice = row['M'].toString().toDouble() +// def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price) +// row['O'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15) +// } } }); } + def addRentRate(){ + def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"] + def radioMap = [:] + for(def city : citys) { + def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1)); + def avg_radio = CommonDatautils.findCityAvgRentRadio(city, lastMonthDay) + radioMap.put(city, avg_radio) + } + + ExcelUtils.multiThreadingModify(path, { row -> + def city = row['G'] + if (row['F'] && row['O'] && !row['P']) { + def avg_rent_radio = radioMap.get(city) + if(avg_rent_radio){ + row['P'] = (int)(row['O']/avg_rent_radio) + row['S'] = "租金推算1" + } + } + }) + } + + def addArroundPrice(){ + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['O']) { + def arroundObj = FangjiaClientUtils.arroundPrice(row['A'], row['E']) + if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) { + def priceObj = arroundObj.result + row['O'] = priceObj.unitPrice +// row['N'] = (int) (priceObj.unitPrice * row['E']) + row['S'] = "周边价格" + } + } + }); + } + def addAuthor(){ - def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx" - def authors = ["王蕾","丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"] - def total = 747 + def authors = ["丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"] def personTotal = authors.size() def avg = Math.ceil(total/personTotal) def index = 0 def count = 0; ExcelUtils.modify(path, { row -> + if(row['B']){ if(index < avg){ - row['P'] = authors[count] + row['N'] = authors[count] } index++ if(index >= avg){ count++ index = 0 } + } }); } @@ -236,4 +318,295 @@ class MatchEvaluateRegionTest { }); return avg_price } + + def addSorce(){ + ExcelUtils.multiThreadingModify(path, { d -> + if(d['F']){ + def score = findDistrictScore(d['F']) + if(score){ + d['R'] = score + } + } + }); + } + + def addArroundScore(){ + ExcelUtils.multiThreadingModify(path, { d -> + if(!d['R'] && d['F']){ + def dist = null + if(d['F'].toString().size()>16){ + dist = CommonDatautils.findDistrictById(d['F']) + } + + if(!dist){ + dist = new BasicDBObject() + dist.city = d['G'] + dist.lng = d['M'].toString().toDouble()*10000000000L + dist.lat = d['N'].toString().toDouble()*10000000000L + } + if(dist.lat){ + List list = DbUtils.findAroundDistrict(dist) + if (list) { + list = sortByDistance(dist, list); + if(list && list.size()>10){ + list = list[0..9] + } + if(list){ + def scores = [] + for(def s : list){ + def score = findDistrictScore(s._id.toString()) + if(score){ + scores << score + } + } + def arround_score =(int)(scores.sum()/scores.size()) + d['R'] = arround_score + d['T'] = "周边小区评分2" + } + } + } + } + }); + } + + def addArroundPriceByLat(){ + ExcelUtils.multiThreadingModify(path, { d -> + if(!d['O'] && d['F'] && (d['K'].contains("住宅") || d['K'].contains("商住") ||d['K'].contains("别墅"))){ + def dist = null + if(d['F'].toString().size()>16){ + dist = CommonDatautils.findDistrictById(d['F']) + } + + if(!dist){ + dist = new BasicDBObject() + dist.city = d['G'] + dist.lng = d['M'].toString().toDouble()*10000000000L + dist.lat = d['N'].toString().toDouble()*10000000000L + } + if(dist.lat){ + List list = DbUtils.findAroundDistrict(dist) + if (list) { + list = sortByDistance(dist, list); + if(list && list.size()>10){ + list = list[0..9] + } + if(list){ + def prices = [] + for(def s : list){ + def priceObj = CommonDatautils.findDistrictPrice(s.city, s.region, s.name) + if(priceObj){ + prices << priceObj.avg_price + } + } + if(prices){ + def arround_price =(int)(prices.sum()/prices.size()) + d['O'] = arround_price + d['S'] = "周边均价3" + } + } + } + } + } + }); + } + + + def sortByDistance(d, list){ + def newList = [] + for(def nb : list){ + def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0) + nb.distance = distance + newList << nb + } + //排序 + Collections.sort(newList, new Comparator() { + @Override + int compare(Object o1, Object o2) { + if(o1.distance > o2.distance) + return 1;//第二个元素(o1)比第一个元素(o2)大,返回-1 + if(o1.distance == o2.distance) + return 0; + return -1; + } + }) + + return newList; + } + + + + def findDistrictScore(id){ + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("_id", id)) + def indexObj = ElasticUtils.executeSimpleQuery("192.168.0.12", "district_comp_potential", queryBuilder); + def score = 0 + for(def h : indexObj){ + score = h.total_price_rate + if(score){ + break; + } + } + return score + } + + + def addLevel() { + def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"]// + def weekend = DateUtils.getFirstDayOfWeek(new Date()); + def cityLevelMap = [:] + for(def city : citys){ + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region","name"]) + def priceMap = CommonDatautils.queryDistrictPrice(city, weekend) + def datas = addPriceData(dists, priceMap); + datas = calcuteLevel(datas); + cityLevelMap.put(city, ["price":priceMap, "level":datas]) + println city +" 楼盘评级同步完成" + } + + ExcelUtils.multiThreadingModify(path, { row -> + + def address = row['E'] + def city = RegionData.formatCity(row['A']) + + if(row['K'] && !row['K'].contains("住宅") && !row['K'].contains("商住") && !row['K'].contains("别墅")){ + if(city){ + def map = cityLevelMap.get(city) + def rMap = null + if(map){ + rMap = getArroundDistrictPriceByAddress(city, address, map.level) + } + + if(rMap){ + if(rMap.rank){ + row['Q'] = rMap.rank + } + } + } + } + }) + } + + def addPriceData(dists, priceMap){ + def newLists = [] + for(def d : dists){ + def price = priceMap.get(d.city+"##"+d.region+"##"+d.name); + if(price){ + d.avg_price = price; + newLists << d; + } + } + if(newLists){ + newLists.sort {a,b-> + def v1 = a.avg_price + def v2 = b.avg_price + return v1.compareTo(v2) + } + } + return newLists; + } + + def calcuteLevel(def datas){ + def total = datas.size() + def v = (int)(total/10) + def start = 0; + def end = 0; + def map = [:] + for(int i=1;i<=10;i++){ + end = end + v; + if(i == 10){ + map.put(i, [start, total]) + }else{ + map.put(i, [start, end]) + } + start = end; + } + for(def d=0;dm && d<=n){ + dist.level = 10 - o.key + 1 + break + } + } + } + } + return datas; + } + + def getArroundDistrictPriceByAddress(def city, def address, def levels){ + def json = GaoDeApiUtils.geo(city, address) + def location = null + def detail = new BasicDBObject() + def level = null + def rank = null + if(json && json.count>=1){ + def m = json.geocodes[0] + if(m){ + location = m.location + level = m.level + } + } + if(location){ + def arrs = location.split(",") + detail.lng = arrs[0].toString().toDouble()*10000000000L + detail.lat = arrs[1].toString().toDouble()*10000000000L + } + if(detail.lng){ + detail.city = city + + List list = DbUtils.findAroundDistrictByRadius(detail, 3000) + if(list && list.size()>10){ + list = sortByDistance(detail, list); + try{ + list = list[0..9] + }catch(Exception e){ + println list.size() + } + + } + def arroundAvgPrice = getArroundAvgPrice(list) + if(arroundAvgPrice){ + + def mod = Integer.MAX_VALUE; + for(def d : levels){ + def newMod = java.lang.Math.abs( d.avg_price - arroundAvgPrice) + if(newMod if(!row['D'] && row['B']){ - def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null) + def jsonObject = OpenApiClient.addressMatch5("重庆", row['B'], "property", false) if(jsonObject != null && jsonObject.precise!=-1) { row['D'] = jsonObject.get("id") row['E'] = jsonObject.get("city") @@ -77,7 +77,7 @@ class MatchEvaluateTest { // def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx" ExcelUtils.multiThreadingModify(path, { row -> if(!row['D'] && row['B']){ - def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], null, true, null, null) + def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], null, true) if(jsonObject != null && jsonObject.precise!=-1) { row['D'] = jsonObject.get("id") row['E'] = jsonObject.get("city") @@ -117,7 +117,7 @@ class MatchEvaluateTest { def addAuthor(){ // def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx" - def authors = ["王蕾","丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"] + def authors = ["丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"] def personTotal = authors.size() def avg = Math.ceil(total/personTotal) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocStat.groovy new file mode 100644 index 000000000..42aedde30 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocStat.groovy @@ -0,0 +1,1221 @@ +package com.fangjia.script.chongqing + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.QueryOperators + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class PocStat { + + + public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", lineUrl) + public static final DBCollection LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia_base", "loudong", lineUrl) + + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) + + static main(args) { + def task = new PocStat(); +// task.regionDistStat(); +// task.sampleStat() +// task.tradeStat() +// task.paimaiStat() +// task.evalStat() +// task.fileldStat() + +// task.removeDelete() + +// task.loudongStat() +// task.houseLoudongStat() + + task.houseLoudongStat2() + +// task.updteHoledCount() + } + + def loudongStat(){ + def retMaps = getAllLouDong() + def total_count = retMaps.keySet().size() + + def all_floorsCount = 0; + def all_buildYearCount = 0 + def all_householdsCount = 0 + def all_elevatorNumberCount = 0 + def all_roomCount = 0 + + def f_all_floorsCount = 0; + def f_all_buildYearCount = 0 + def f_all_householdsCount = 0 + def f_all_elevatorNumberCount = 0 + def f_all_roomCount = 0 + + + def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"] + + def z_count = 0 + def f_count = 0 + + retMaps.each{key,lists-> + def floorsCount = 0; + def buildYearCount = 0 + def householdsCount = 0 + def elevatorNumberCount = 0 + def roomCount = 0 + + def f_floorsCount = 0; + def f_buildYearCount = 0 + def f_householdsCount = 0 + def f_elevatorNumberCount = 0 + def f_roomCount = 0 + + def dist = CommonDatautils.findDistrictById(key) + + if(zRegions.contains(dist.region)){ + z_count++ + }else{ + f_count++ + } + + for(def row : lists){ + + if(zRegions.contains(dist.region)){ + + if(row.floors){ + floorsCount++ + } + if(dist.finish_building_date){ + buildYearCount++ + } + if(row.households){ + householdsCount++ + }else{ + if(row.room_count && row.floors){ + householdsCount++ + } + } + if(row.elevatorNumber && row.elevatorNumber>0){ + elevatorNumberCount++ + } + + if(row.room_count>0){ + roomCount++ + } + + }else{ + if(row.floors){ + f_floorsCount++ + } + if(dist.finish_building_date){ + f_buildYearCount++ + } + if(row.households){ + f_householdsCount++ + }else{ + if(row.room_count && row.floors){ + f_householdsCount++ + } + } + if(row.elevatorNumber && row.elevatorNumber>0){ + f_elevatorNumberCount++ + } + + if(row.room_count>0){ + f_roomCount++ + } + } + + } + + def dist_total_count = lists.size()*0.6 + if(floorsCount >= dist_total_count){ + all_floorsCount ++ + } + + if(buildYearCount >= dist_total_count){ + all_buildYearCount ++ + } + + if(householdsCount >= dist_total_count){ + all_householdsCount ++ + } + + if(elevatorNumberCount >= dist_total_count){ + all_elevatorNumberCount ++ + } + + if(roomCount >= dist_total_count){ + all_roomCount ++ + } + + + if(f_floorsCount >= dist_total_count){ + f_all_floorsCount ++ + } + + if(f_buildYearCount >= dist_total_count){ + f_all_buildYearCount ++ + } + + if(f_householdsCount >= dist_total_count){ + f_all_householdsCount ++ + } + + if(f_elevatorNumberCount >= dist_total_count){ + f_all_elevatorNumberCount ++ + } + if(f_roomCount >= dist_total_count){ + f_all_roomCount ++ + } + } + def floor_prec = DataHandleUtils.getDoubleRound(2, all_floorsCount*100/z_count) + def year_prec = DataHandleUtils.getDoubleRound(2, all_buildYearCount*100/z_count) + def households_prec = DataHandleUtils.getDoubleRound(2, all_householdsCount*100/z_count) + def elevatorNum_prec = DataHandleUtils.getDoubleRound(2, all_elevatorNumberCount*100/z_count) + def roomcount_prec = DataHandleUtils.getDoubleRound(2, all_roomCount*100/z_count) + + + def f_floor_prec = DataHandleUtils.getDoubleRound(2, f_all_floorsCount*100/f_count) + def f_year_prec = DataHandleUtils.getDoubleRound(2, f_all_buildYearCount*100/f_count) + def f_households_prec = DataHandleUtils.getDoubleRound(2, f_all_householdsCount*100/f_count) + def f_elevatorNum_prec = DataHandleUtils.getDoubleRound(2, f_all_elevatorNumberCount*100/f_count) + def f_roomcount_prec = DataHandleUtils.getDoubleRound(2, f_all_roomCount*100/f_count) + + println "floor "+ floor_prec + println "year "+ year_prec + println "households "+ households_prec + println "elevator "+ elevatorNum_prec + + def rets = [] + rets << ["城区","楼栋号","楼栋ID","物业类型","主朝向","地上层数","地下层数","竣工日期","梯","户","总户数","景观"] + rets << ["主城区",100,100,100,0,floor_prec,0,year_prec,elevatorNum_prec,roomcount_prec,households_prec,0] + rets << ["非主城区",100,100,100,0,f_floor_prec,0,f_year_prec,f_elevatorNum_prec,f_roomcount_prec,f_households_prec,0] + ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx",rets, "楼栋") + } + + + def houseLoudongStat2(){ + def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"] + def dists = CommonDatautils.findAllDistrictByCity("重庆","district", ["city","region","name"]) + def z_count = 0 + def f_count = 0 + + def z_directionCount = 0 + def f_directionCount = 0 + + def f_sightCount = 0 + def z_sightCount = 0 + def index = 0 + for(def dist : dists){ + index ++ + def rets = getAllHouseLoudong2(dist._id.toString()) + println index +" "+rets.size() + if(rets){ + def directionCount = 0 + def sightCount = 0 + for(def d : rets){ + if(d.direction){ + directionCount ++ + } + if(d.sight){ + sightCount ++ + } + } + + def total = rets.size()*0.6 + if(zRegions.contains(dist.region)){ + z_count++ + + if(directionCount>=total){ + z_directionCount++ + } + if(sightCount >=total){ + z_sightCount++ + } + }else{ + f_count++ + + if(directionCount>=total){ + f_directionCount++ + } + if(sightCount >=total){ + f_sightCount++ + } + } + } + } + + def z_direct_prec = DataHandleUtils.getDoubleRound(2, z_directionCount*100/z_count) + def z_sight_prec = DataHandleUtils.getDoubleRound(2, z_sightCount*100/z_count) + + def f_direct_prec = DataHandleUtils.getDoubleRound(2, f_directionCount*100/f_count) + def f_sight_prec = DataHandleUtils.getDoubleRound(2, f_sightCount*100/f_count) + def rets = [] + rets << ["城区","户号","户ID","主朝向","景观","户型结构"] + rets << ["主城区",100,100,z_direct_prec, z_sight_prec, 100] + rets << ["非主城区",100,100, f_direct_prec, f_sight_prec, 100] + ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx",rets, "户1") + } + + def houseLoudongStat(){ + def map = getAllHouseLoudong() + def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"] + + def z_count = 0 + def f_count = 0 + + def z_directionCount = 0 + def f_directionCount = 0 + + def f_sightCount = 0 + def z_sightCount = 0 + map.each{key,list-> + def dist = CommonDatautils.findDistrictById(key) + def directionCount = 0 + def sightCount = 0 + for(def d : list){ + if(d.direction){ + directionCount ++ + } + if(d.sight){ + sightCount ++ + } + } + + def total = list.size()*0.6 + if(zRegions.contains(dist.region)){ + z_count++ + + if(directionCount>=total){ + z_directionCount++ + } + if(sightCount >=total){ + z_sightCount++ + } + }else{ + f_count++ + + if(directionCount>=total){ + f_directionCount++ + } + if(sightCount >=total){ + f_sightCount++ + } + } + } + + def z_direct_prec = DataHandleUtils.getDoubleRound(2, z_directionCount*100/z_count) + def z_sight_prec = DataHandleUtils.getDoubleRound(2, z_sightCount*100/z_count) + + def f_direct_prec = DataHandleUtils.getDoubleRound(2, f_directionCount*100/f_count) + def f_sight_prec = DataHandleUtils.getDoubleRound(2, f_sightCount*100/f_count) + def rets = [] + rets << ["城区","户号","户ID","主朝向","景观","户型结构"] + rets << ["主城区",100,100,z_direct_prec, z_sight_prec, 100] + rets << ["非主城区",100,100, f_direct_prec, f_sight_prec, 100] + ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx",rets, "户") + } + + def getAllHouseLoudong(){ + def q = new BasicDBObject() + q.put("city","重庆") + q.put("district_id", new BasicDBObject(QueryOperators.EXISTS, true)) + + def keys = new BasicDBObject() + keys.put("district_id",1) + keys.put("direction",1) + keys.put("sight",1) + def dbcursor = HOUSE_LOUDONG_COLL.find(q).batchSize(5000) + def retMap = [:] + for(def d : dbcursor){ + def districtId = d.district_id + if(districtId){ + def lists = retMap.get(districtId) + if(!lists){ + lists = [] + retMap.put(districtId, lists) + } + lists << d + } + } + return retMap; + } + + + def getAllHouseLoudong2(districtId){ + def q = new BasicDBObject() + q.put("city","重庆") + q.put("district_id", districtId) + + def keys = new BasicDBObject() + keys.put("district_id",1) + keys.put("direction",1) + keys.put("sight",1) + def dbcursor = HOUSE_LOUDONG_COLL.find(q).batchSize(5000) + def lists = [] + for(def d : dbcursor){ + lists << d + } + return lists; + } + + def getAllLouDong(){ + def q = new BasicDBObject() + q.put("city","重庆") + q.put("id_in_seaweed", new BasicDBObject(QueryOperators.EXISTS, true)) + def dbcursor = LOUDONG_LINE_COLL.find(q).batchSize(5000) + def retMap = [:] + for(def d : dbcursor){ + def districtId = d.id_in_seaweed + if(districtId){ + def lists = retMap.get(districtId) + if(!lists){ + lists = [] + retMap.put(districtId, lists) + } + lists << d + } + } + return retMap; + } + + def updteHoledCount(){ + def datas = findAllLouDong() + def totalCount = new AtomicLong() + println datas.size() + final CountDownLatch latch = new CountDownLatch(datas.size()) + for(def i=0;i2){ + q.put("unit_num", arrs[2]) + } + + def udata = new BasicDBObject() + udata.put("room_count", max_room) + LOUDONG_LINE_COLL.update(q, new BasicDBObject('$set', udata), false, true); + } + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + }; + }) + } + latch.await() + } + + def findAllLouDong(){ + def q = new BasicDBObject() + q.put("city","重庆") + q.put("id_in_seaweed", new BasicDBObject(QueryOperators.EXISTS, true)) + def dbcursor = LOUDONG_LINE_COLL.find(q) + def keys = new HashSet() + for(def d : dbcursor){ + def districtId = d.id_in_seaweed + def key = districtId +"##"+d.house_num + if(d.unit_num && d.unit_num!="0"){ + key = key + "##"+d.unit_num + } + if(!keys.contains(key)){ + keys << key + } + } + return keys + } + + def findHouseLouDongRoom(def arrs){ + def q = new BasicDBObject() + q.put("city","重庆") + q.put("district_id", arrs[0]) + q.put("house_num", arrs[1]) + if(arrs.size()>2){ + q.put("unit_num", arrs[2]) + } + def dbcursor = HOUSE_LOUDONG_COLL.find(q) + def max_room = 0 + for(def d : dbcursor){ + def room = null + if(d.room_num && d.room_num.contains("-")){ + room = d.room_num.substring(d.room_num.indexOf("-")+1, d.room_num.length()) + }else if(d.room_num && d.room_num.contains("0")){ + room = d.room_num.substring(d.room_num.toString().lastIndexOf("0")+1, d.room_num.length()) + } + if(room && room.isNumber()){ + if(room.toInteger()>max_room){ + max_room = room.toInteger() + } + } + } + return max_room; + } + def removeDelete(){ + def dists = CommonDatautils.findAllDistrictByCity("重庆","district", ["city","region","name", "name_cat"]) + for(def obj : dists){ + def name_cat = obj.name_cat; + if(name_cat==12){ + def count = CommonDatautils.findDistrictCount(obj.city, obj.region, obj.name, "district") + if(count>1){ + println obj.name +" "+count + DbUtils.seaweed.remove(new BasicDBObject("_id", obj._id)) + } + } + } + } + + def regionDistStat(){ + def city = "重庆" + def regionMap = CommonDatautils.getRegionsByCity(city) + def set =regionMap.keySet(); + def rows = [] + rows << ["区域","小区量"] + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name"]) + for(def region : set){ + def count = CommonDatautils.findDistrictCountByRegionBlock(city, region, null, "district") + +// def regionCount = 0 +// def regionHouseLDCount = 0 +// for(def d : dists){ +// if(d.region == region){ +// def loudongCount = CommonDatautils.findLoudong(d._id.toString()) +// if(loudongCount && loudongCount>0){ +// regionCount = regionCount+ loudongCount +// } +// +// def houseLouDongCount = CommonDatautils.findHouseLoudong(d._id.toString()) +// if(houseLouDongCount && houseLouDongCount>0){ +// regionHouseLDCount = regionHouseLDCount + houseLouDongCount +// } +// } +// } + rows << [region, count] + println region +" 完成" + } + ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx",rows, "各区楼盘量统计1") + } + + def sampleStat(){ + def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"]) + + final CountDownLatch latch = new CountDownLatch(dists.size()) + + def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"] + def zCityDistTotal = new AtomicLong() + def fCityDistTotal = new AtomicLong() + //[0, 1), [1, 5), [5, 20), [20, 50), [50,+) + def zHLevel1 = new AtomicLong() + def zHLevel2 = new AtomicLong() + def zHLevel3 = new AtomicLong() + def zHLevel4 = new AtomicLong() + def zHLevel5 = new AtomicLong() + + def fHLevel1 = new AtomicLong() + def fHLevel2 = new AtomicLong() + def fHLevel3 = new AtomicLong() + def fHLevel4 = new AtomicLong() + def fHLevel5 = new AtomicLong() + + def rets = [] + rets << ["城区","小区量","[0, 1)","[1, 5)","[5, 20)","[20, 50)","[50,+)"] + + + def index = new AtomicLong() + for(def i=0;i=50){ + zHLevel5.addAndGet(1) + } + } + }else{ + fCityDistTotal.addAndGet(1) + + if(results){ + if(results.size()<5){ + fHLevel2.addAndGet(1) + }else if(results.size()<20){ + fHLevel3.addAndGet(1) + }else if(results.size()<50){ + fHLevel4.addAndGet(1) + }else if(results.size()>=50){ + fHLevel5.addAndGet(1) + } + } + } + }else{ + if(zRegions.contains(d.region)) { + zCityDistTotal.addAndGet(1) + zHLevel1.addAndGet(1) + + println d.city+"#"+d.region+"#"+d.name + }else{ + fCityDistTotal.addAndGet(1) + fHLevel1.addAndGet(1) + } + } + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + latch.await() + def ret1 = ["主城区", zCityDistTotal.get(), zHLevel1.get(), zHLevel2.get(), zHLevel3.get(), zHLevel4.get(), zHLevel5.get()] + def ret2 = ["非主城区", fCityDistTotal.get(), fHLevel1.get(), fHLevel2.get(), fHLevel3.get(), fHLevel4.get(), fHLevel5.get()] + rets.add(ret1) + rets.add(ret2) + ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx", rets, "案例数量分布-样本") + } + + + def tradeStat(){ + def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"]) + + final CountDownLatch latch = new CountDownLatch(dists.size()) + + def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"] + def zCityDistTotal = new AtomicLong() + def fCityDistTotal = new AtomicLong() + //[0, 1), [1, 5), [5, 20), [20, 50), [50,+) + def zTLevel1 = new AtomicLong() + def zTLevel2 = new AtomicLong() + def zTLevel3 = new AtomicLong() + def zTLevel4 = new AtomicLong() + def zTLevel5 = new AtomicLong() + + def fTLevel1 = new AtomicLong() + def fTLevel2 = new AtomicLong() + def fTLevel3 = new AtomicLong() + def fTLevel4 = new AtomicLong() + def fTLevel5 = new AtomicLong() + + def rets = [] + rets << ["城区","小区量","[0, 1)","[1, 5)","[5, 20)","[20, 50)","[50,+)"] + + + def index = new AtomicLong() + for(def i=0;i=50){ + zTLevel5.addAndGet(1) + } + } + }else{ + fCityDistTotal.addAndGet(1) + + if(results){ + if(results.size()<5){ + fTLevel2.addAndGet(1) + }else if(results.size()<20){ + fTLevel3.addAndGet(1) + }else if(results.size()<50){ + fTLevel4.addAndGet(1) + }else if(results.size()>=50){ + fTLevel5.addAndGet(1) + } + } + } + }else{ + if(zRegions.contains(d.region)) { + zCityDistTotal.addAndGet(1) + zTLevel1.addAndGet(1) + }else{ + fCityDistTotal.addAndGet(1) + fTLevel1.addAndGet(1) + } + } + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + latch.await() + def ret1 = ["主城区", zCityDistTotal.get(), zTLevel1.get(), zTLevel2.get(), zTLevel3.get(), zTLevel4.get(), zTLevel5.get()] + def ret2 = ["非主城区", fCityDistTotal.get(), fTLevel1.get(), fTLevel2.get(), fTLevel3.get(), fTLevel4.get(), fTLevel5.get()] + rets.add(ret1) + rets.add(ret2) + ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx", rets, "案例数量分布-成交") + println "成交案例统计完成" + } + + def paimaiStat(){ + def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"]) + + final CountDownLatch latch = new CountDownLatch(dists.size()) + + def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"] + def zCityDistTotal = new AtomicLong() + def fCityDistTotal = new AtomicLong() + //[0, 1), [1, 5), [5, 20), [20, 50), [50,+) + def zPLevel1 = new AtomicLong() + def zPLevel2 = new AtomicLong() + def zPLevel3 = new AtomicLong() + def zPLevel4 = new AtomicLong() + def zPLevel5 = new AtomicLong() + + def fPLevel1 = new AtomicLong() + def fPLevel2 = new AtomicLong() + def fPLevel3 = new AtomicLong() + def fPLevel4 = new AtomicLong() + def fPLevel5 = new AtomicLong() + + def rets = [] + rets << ["城区","小区量","[0, 1)","[1, 5)","[5, 20)","[20, 50)","[50,+)"] + + + def index = new AtomicLong() + for(def i=0;i=50){ + zPLevel5.addAndGet(1) + } + } + }else{ + fCityDistTotal.addAndGet(1) + + if(results){ + if(results.size()<5){ + fPLevel2.addAndGet(1) + }else if(results.size()<20){ + fPLevel3.addAndGet(1) + }else if(results.size()<50){ + fPLevel4.addAndGet(1) + }else if(results.size()>=50){ + fPLevel5.addAndGet(1) + } + } + } + }else{ + if(zRegions.contains(d.region)) { + zCityDistTotal.addAndGet(1) + zPLevel1.addAndGet(1) + }else{ + fCityDistTotal.addAndGet(1) + fPLevel1.addAndGet(1) + } + } + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + latch.await() + def ret1 = ["主城区", zCityDistTotal.get(), zPLevel1.get(), zPLevel2.get(), zPLevel3.get(), zPLevel4.get(), zPLevel5.get()] + def ret2 = ["非主城区", fCityDistTotal.get(), fPLevel1.get(), fPLevel2.get(), fPLevel3.get(), fPLevel4.get(), fPLevel5.get()] + rets.add(ret1) + rets.add(ret2) + ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx", rets, "案例数量分布-拍卖") + println "拍卖案例统计完成" + } + def evalStat(){ + def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"]) + + final CountDownLatch latch = new CountDownLatch(dists.size()) + + def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"] + def zCityDistTotal = new AtomicLong() + def fCityDistTotal = new AtomicLong() + //[0, 1), [1, 5), [5, 20), [20, 50), [50,+) + def zELevel1 = new AtomicLong() + def zELevel2 = new AtomicLong() + def zELevel3 = new AtomicLong() + def zELevel4 = new AtomicLong() + def zELevel5 = new AtomicLong() + + def fELevel1 = new AtomicLong() + def fELevel2 = new AtomicLong() + def fELevel3 = new AtomicLong() + def fELevel4 = new AtomicLong() + def fELevel5 = new AtomicLong() + + def rets = [] + rets << ["城区","小区量","[0, 1)","[1, 5)","[5, 20)","[20, 50)","[50,+)"] + + + def index = new AtomicLong() + for(def i=0;i=50){ + zELevel5.addAndGet(1) + } + } + }else{ + fCityDistTotal.addAndGet(1) + + if(results){ + if(results.size()<5){ + fELevel2.addAndGet(1) + }else if(results.size()<20){ + fELevel3.addAndGet(1) + }else if(results.size()<50){ + fELevel4.addAndGet(1) + }else if(results.size()>=50){ + fELevel5.addAndGet(1) + } + } + } + }else{ + if(zRegions.contains(d.region)) { + zCityDistTotal.addAndGet(1) + zELevel1.addAndGet(1) + }else{ + fCityDistTotal.addAndGet(1) + fELevel1.addAndGet(1) + } + } + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + latch.await() + def ret1 = ["主城区", zCityDistTotal.get(), zELevel1.get(), zELevel2.get(), zELevel3.get(), zELevel4.get(), zELevel5.get()] + def ret2 = ["非主城区", fCityDistTotal.get(), fELevel1.get(), fELevel2.get(), fELevel3.get(), fELevel4.get(), fELevel5.get()] + rets.add(ret1) + rets.add(ret2) + ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx", rets, "案例数量分布-评估") + println "评估案例统计完成" + } + + def fileldStat(){ + def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"]) + final CountDownLatch latch = new CountDownLatch(dists.size()) + def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"] + def alias_count = new AtomicLong() + def address_count = new AtomicLong() + def block_count = new AtomicLong() + def township_count = new AtomicLong() + def price_count = new AtomicLong() + def lng_count = new AtomicLong() + def finishBuildTime_count = new AtomicLong() + def developer_count = new AtomicLong() + def green_count = new AtomicLong() + def ratio_count = new AtomicLong() + def carsite_count = new AtomicLong() + def propertyManagement_count = new AtomicLong() + def managementFee_count = new AtomicLong() + def building_count = new AtomicLong() + def holed_count = new AtomicLong() + def car_holed_count = new AtomicLong() + def all_area_count = new AtomicLong() + def key_school_count = new AtomicLong() + def school_count = new AtomicLong() + def traffic_count = new AtomicLong() + def market_count = new AtomicLong() + def hospital_count = new AtomicLong() + def bank_count = new AtomicLong() + + def f_alias_count = new AtomicLong() + def f_address_count = new AtomicLong() + def f_block_count = new AtomicLong() + def f_township_count = new AtomicLong() + def f_price_count = new AtomicLong() + def f_lng_count = new AtomicLong() + def f_finishBuildTime_count = new AtomicLong() + def f_developer_count = new AtomicLong() + def f_green_count = new AtomicLong() + def f_ratio_count = new AtomicLong() + def f_carsite_count = new AtomicLong() + def f_propertyManagement_count = new AtomicLong() + def f_managementFee_count = new AtomicLong() + def f_building_count = new AtomicLong() + def f_holed_count = new AtomicLong() + def f_car_holed_count = new AtomicLong() + def f_all_area_count = new AtomicLong() + def f_key_school_count = new AtomicLong() + def f_school_count = new AtomicLong() + def f_traffic_count = new AtomicLong() + def f_market_count = new AtomicLong() + def f_hospital_count = new AtomicLong() + def f_bank_count = new AtomicLong() + + def zCityDistTotal = new AtomicLong() + def fCityDistTotal = new AtomicLong() + def index = new AtomicLong() + for(def i=0;i + if(row['F'] && row['R']){ + def avg_price = findAvgPrice(row['R']) + if(avg_price){ + def ppe15 = Math.abs((row['F'] - avg_price) * 100 / avg_price) + row['Z'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15) + } + } + }); + } + + def pkPrice(path){ + ExcelUtils.multiThreadingModify(path, { row -> + def commid = row['R'] + if(commid){ + def dist = CommonDatautils.findDistrictById(commid.trim()) + if(dist){ + def jsonObject = OpenApiClient.webankEvaluate(dist.city, dist.region, dist.name, row['E'].toString()) + if(jsonObject != null && jsonObject.code==200) { + def result = jsonObject.result + if(result){ + row['Y'] = result.get("unitPrice") + } + } + } + } + }) + } + + + def match(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['Q'] && row['D']){ + def city = row['B'] + def address = row['D'] + def region = row['C'] + println "[match] city:"+city+" region:"+region+" address:"+address + def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, null) + if(jsonObject != null && jsonObject.precise!=-1) { + row['Q'] = jsonObject.get("name") + row['R'] = jsonObject.get("id") + } + } + }) + } + + def matchMap(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['Q'] && row['D']){ + def city = row['B'] + def address = row['D'] + def region = row['C'] + println "city:"+city+" region:"+region+" address:"+address + def jsonObject = OpenApiClient.addressMatch(city, address, null, true, region, null) + if(jsonObject != null && jsonObject.precise!=-1) { + def propertyType = jsonObject.get("propertyType") + if(propertyType && (propertyType.contains("住宅") || propertyType.contains("别墅") || propertyType.contains("商住"))){ + row['Q'] = jsonObject.get("name") + row['R'] = new ObjectId().toString() + row['W'] = propertyType + + def arroundObj = FangjiaClientUtils.arroundPrice(city, address) + if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) { + def priceObj = arroundObj.result + row['F'] = priceObj.unitPrice + } + } + + } + } + }) + } + + def addPrice(path){ + ExcelUtils.multiThreadingModify(path, { row -> + def commid = row['R'] + if(commid){ + def dist = CommonDatautils.findDistrictById(commid.trim()) + if(dist){ + row['Q'] = dist.name + def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name) + if(priceObj){ + row['F'] = priceObj.avg_price + } + } + } + }) + } + + + + def addArroundPrice(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['F']) { + def arroundObj = FangjiaClientUtils.arroundPrice(row['B'], row['D']) + if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) { + def priceObj = arroundObj.result + row['F'] = priceObj.unitPrice + row['W'] = "周边均价" + } + } + }); + } + + def standard(path){ + ExcelUtils.multiThreadingModify(path, { row -> + def city = row['B'] + def address = row['D'] + if(address){ + println "[standard] address:"+address + def jsonObject = OpenApiClient.standard2Action(city, address) + if(jsonObject){ + println jsonObject + row['G'] = jsonObject.get("province") + row['H'] = jsonObject.get("city") + row['I'] = jsonObject.get("district") + row['J'] = jsonObject.get("township") + row['K'] = jsonObject.get("street") + row['L'] = jsonObject.get("streetNumber") + row['M'] = jsonObject.get("buildingNumber") + row['N'] = jsonObject.get("unitNumber") + row['O'] = jsonObject.get("floor") + row['P'] = jsonObject.get("roomNumber") + } + } + + }) + } + + def addLoudongNum(path){ + + ExcelUtils.multiThreadingModify(path, { row -> + def commid = row['R'] + def houseNum = row['M'] + def unitNum = row['N'] + println "[addLoudongNum] commid:"+commid+" houseNum:"+houseNum + if (commid && houseNum) { + def jsonObject = OpenApiClient.searchLoudong(commid, houseNum, unitNum) + if (jsonObject && jsonObject.result) { + if(jsonObject.result.size()>=1){ + def obj = jsonObject.result[0] + row['T'] = obj.id + row['S'] = obj.houseNumber + } + } + + if(row['P']){ + def jsonObject1 = OpenApiClient.searchHouseLoudong(commid, houseNum, unitNum, row['P']) + if (jsonObject1 && jsonObject1.result) { + if(jsonObject1.result.size()>=1){ + def obj = jsonObject1.result[0] + row['V'] = obj.id + row['U'] = obj.roomNum + } + } + } + } + + }) + } + + def createLoudong(path, output){ + def rets = new CopyOnWriteArrayList() + def datas = ExcelUtils.read(path, 0, 1) + rets << ["输入城市","输入区域","输入地址","面积","楼栋号","楼栋ID","物业类型","主朝向","地上层数","地下层数","竣工日期","梯","户","总户数","景观","户号","户ID","主朝向","景观","户型结构"] + for(def row : datas){ + def data = [] + data << row['B'] + data << row['C'] + data << row['D'] + data << row['E'] + if (row['R'] && row['M']) { + def jsonObject = OpenApiClient.searchLoudong(row['R'], row['M'], row['N']) + if (jsonObject && jsonObject.result && jsonObject.result.size()>=1) { + def obj = jsonObject.result[0] + data << obj.houseNumber + data << obj.id + data << obj.type// 物业类型 + data << ""//主朝向 + data << obj.floors//地上层数 + data << ""//地下层数 + data << obj.buildYear;//竣工日期 + data << ""//梯 + data << ""//户 + data << obj.households//总户数 + data << ""//景观 + }else{ + data << "" + data << "" + data << ""// 物业类型 + data << ""//主朝向 + data << ""//地上层数 + data << ""//地下层数 + data << ""//竣工日期 + data << ""//梯 + data << ""//户 + data << ""//总户数 + data << ""//景观 + } + + if(row['P']){ + def jsonObject1 = OpenApiClient.searchHouseLoudong(row['R'], row['M'], row['N'], row['P']) + if (jsonObject1 && jsonObject1.result) { + if(jsonObject1.result.size()>=1){ + def obj = jsonObject1.result[0] + + data << obj.roomNum//户号 + data << obj.id//户ID + data << obj.direction//主 + data << obj.sight//景观 + if(obj.roomNum && obj.roomNum.contains("跃")){ + data << "跃式"//户型结构 + }else{ + data << ""//户型结构 + } + + } + } + } + } + rets << data + } + ExcelUtils.write(output, rets, "底层数据-楼栋-户") + } + + def addDetail(path, output){ + def datas = ExcelUtils.read(path, 0, 1) + def rows = [] + rows << ["输入城市","输入区域","输入地址","小区名称","小区ID","小区别名","小区地址","板块","街道","基准价","经纬度","竣工时间","开发商名称","绿化率","容积率","车位个数","物业公司","物业费","住宅幢数","总户数","车位配比","占地面积","是否学区","周边教育","周边交通","周边商业","周边医院","周边金融机构"] + for(def d : datas){ + def commid = d['R'] + println "[addDetail] commid:"+commid + if(commid){ + def dist = CommonDatautils.findDistrictById(commid) + if(dist){ + def row = [] + def jsonObject = OpenApiClient.districtDetail(dist.city, dist.region, dist.name) + if(jsonObject) { + + row << d['B'] + row << d['C'] + row << d['D'] + row << jsonObject.get("name")//小区名称 + row << commid//小区ID + if(jsonObject.get("alias")){ + row << jsonObject.get("alias").join(",").replace('"',"")//小区别名 + }else{ + row << "" + } + + row << jsonObject.get("address")//小区地址 + row << jsonObject.get("commercialArea")//板块 + row << jsonObject.get("township")//街道*********** + row << jsonObject.get("avgPrice")//基准价 + row << jsonObject.get("lng").toString()+","+jsonObject.get("lat").toString()//经纬度 + row << jsonObject.get("finishBuildTime")//竣工时间 + row << jsonObject.get("developer")//开发商名称 + row << SeaweedUtils.formatVirescencePercent(jsonObject.get("greenPercentage"))//绿化率 + row << SeaweedUtils.formatCubagePercent(jsonObject.get("plotRatio"))//容积率 + row << SeaweedUtils.formatCarSiteCount(jsonObject.get("parking"))//车位个数 + row << jsonObject.get("propertyManagement")//物业公司 + row << jsonObject.get("managementFee")//物业费 + row << jsonObject.get("buildingCount")//住宅幢数 + row << SeaweedUtils.formatHouseholdCount(jsonObject.get("totalUnitCount"))//总户数 + row << SeaweedUtils.calcCarAndHouseholdRadio(jsonObject.get("parking"), jsonObject.get("totalUnitCount"))//车位配比********* + row << SeaweedUtils.formatAllArea(jsonObject.get("floorSpace"))//占地面积 + + } + + def facilityObj = OpenApiClient.facilitys(dist.city, dist.region, dist.name) + if(facilityObj && facilityObj.result){ + def result = facilityObj.result + if(result.keySchool){ + row << '是' + }else { + row << "" + } + + if(result.school){ + row << result.school.join(",").replace('"',"") + }else { + row << "" + } + + def bus = [] + if(result.bus){ + bus.addAll(result.bus) + } + if(result.metro){ + bus.addAll(result.metro) + } + + if(bus){ + row << bus.join(",").replace('"',"") + }else{ + row << "" + } + + if(result.supermarket){ + row << result.supermarket.join(",").replace('"',"") + }else{ + row << "" + } + + if(result.hospital){ + row << result.hospital.join(",").replace('"',"") + }else{ + row << "" + } + + if(result.bank){ + row << result.bank.join(",").replace('"',"") + }else{ + row << "" + } + } + rows << row + } + } + } + ExcelUtils.write(output, rows, "底层数据-小区") + } + + def createPaimai(path, output){ + def datas = ExcelUtils.read(path, 0, 1) + def rows = [] + rows << ["输入城市","输入区域","输入地址","小区名称","标的名称","楼栋","户号","面积","案例时间","类型","状态","成交单价","评估单价"] + def keys = [] + for(def d : datas){ + def commid = d['R'] + println "[createPaimai] commid:"+commid + if(commid) { + def dist = CommonDatautils.findDistrictById(commid) + if (dist) { + def key = dist.city+"#"+dist.region+"#"+dist.name + if(keys.contains(key)){ + continue + } + keys << key + def jsonObject = OpenApiClient.searchPaimais(dist.city, dist.region, dist.name,"20220217",100) + if(jsonObject && jsonObject.result){ + def results = jsonObject.result + for(def r: results){ + def row = [] + row << d['B'] + row << d['C'] + row << d['D'] + row << r.matchName + row << r.address + row << r.houseNum + row << r.roomNum + row << r.area + if(r.startDate){ + row << r.startDate.substring(0, r.startDate.indexOf("T")) + }else{ + row << "" + } + + row << r.count + row << r.status + row << r.tradeAvgPrice + row << r.tradePrice + row << r.id + rows << row + } + println dist.name +" "+results.size() + } + } + } + } + ExcelUtils.write(output, rows, "拍卖案例") + } + + def createHouse(path, output){ + def datas = ExcelUtils.read(path, 0, 1) + def rows = [] + rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"] + def keys = [] + for(def d : datas) { + def commid = d['R'] +// println "[createHouse] commid:"+commid + if (commid) { + def key = d['Q'] + if (keys.contains(key)) { + continue + } + keys << key + def jsonObject = OpenApiClient.houseList(commid,"20210501",20) + if(jsonObject && jsonObject.result) { + def results = jsonObject.result + for (def r : results) { + def row = [] + row << "挂牌" + row << r.districtName + if(r.estateType){ + row << r.estateType + }else{ + if(r.purpose){ + row << r.purpose + }else{ + row << "" + } + } + + if(r.room){ + def roomStr = r.room+"室" + if(r.hall){ + roomStr = roomStr +r.hall+"厅" + } + row << roomStr + }else{ + row << "" + } + row << r.area + row << r.fitment + row << r.pDate + row << "" + row << r.avgPrice + row << r.totalPrice + if(r.floor){ + row << r.floor + }else{ + row << "" + } + if(r.height){ + row << r.height + }else{ + row << "" + } + + row << r.direction + row << SourceNameUtil.getResourceName(r.root) + rows << row + } + + println d['R'] +" "+results.size() + } + } + } + ExcelUtils.write(output, rows, "挂牌案例") + } + + def createEvaluation(path, output){ + def datas = ExcelUtils.read(path, 0, 1) + def rows = [] + rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"] + def keys = [] + for(def d : datas) { + def commid= d['R'] + println "[createEvaluation] commid:"+commid + if (commid) { + if (keys.contains(commid)) { + continue + } + keys << commid + def jsonObject = OpenApiClient.evaluaionList(commid,"20210501",20) + if(jsonObject && jsonObject.result) { + def results = jsonObject.result + for (def r : results) { + def row = [] + row << "评估" + row << r.commName + row << "" + + if(r.roomcnt){ + def roomStr = r.roomcnt+"室" + if(r.livingroomcnt){ + roomStr = roomStr +r.livingroomcnt+"厅" + } + row << roomStr + }else{ + row << "" + } + row << r.buildArea + row << r.fitment + row << r.cDate.substring(0, r.cDate.indexOf("T")) + row << "" + row << r.unitPrice + row << r.totalPrice + if(r.floor){ + row << r.floor + }else{ + row << "" + } + if(r.floorcnt){ + row << r.floorcnt + }else{ + row << "" + } + + if(r.houseaspect && !r.houseaspect.contains("暂")){ + row << r.houseaspect + }else{ + row << "" + } + + row << "房价网" + rows << row + } + + println d['R'] +" "+results.size() + } + } + } + ExcelUtils.write(output, rows, "评估案例") + } + + + def createDealPrice(path, output){ + def datas = ExcelUtils.read(path, 0, 1) + def rows = [] + rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"] + def keys = [] + for(def d : datas) { + def commid= d['R'] + println "[createDealPrice] commid:"+commid + if (commid) { + if (keys.contains(commid)) { + continue + } + keys << commid + def jsonObject = OpenApiClient.searchDealPrice(commid,"20210501",20) + if(jsonObject && jsonObject.result) { + def results = jsonObject.result + for (def r : results) { + def row = [] + row << "成交" + row << r.districtName + if(r.type){ + row << r.type + }else{ + row << "" + } + + if(r.room){ + def roomStr = r.room+"室" + if(r.hall){ + roomStr = roomStr +r.hall+"厅" + } + row << roomStr + }else{ + row << "" + } + row << r.area + row << r.fitment + row << r.tradeDate.substring(0, r.tradeDate.indexOf("T")) + row << "" + row << r.avgPrice + row << r.totalPrice + if(r.floor){ + row << r.floor + }else{ + row << "" + } + if(r.height){ + row << r.height + }else{ + row << "" + } + + if(r.direction && !r.direction.contains("暂")){ + row << r.direction + }else{ + row << "" + } + + row << r.source + rows << row + } + + println d['R'] +" "+results.size() + } + } + } + ExcelUtils.write(output, rows, "成交案例") + } + + def addAuthor(def path){ + def authors = ["丁世菁","王文艳","董彦均","曹玉彦","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","穆良书","沈燕芳"] + def personTotal = authors.size() + def avg = Math.ceil(total/personTotal) + + def index = 0 + def count = 0; + ExcelUtils.modify(path, { row -> + if(row['B']){ + if(index < avg){ + row['X'] = authors[count] + } + index++ + if(index >= avg){ + count++ + index = 0 + } + } + }); + } + + def findAvgPrice(def commid){ + def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2"; + def avg_price = null + sqlInstance.eachRow(sql, { r -> + avg_price = r.target_avg_price + }); + return avg_price + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index d6ab63366..b8e3cd5a1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -68,6 +68,10 @@ class CommonDatautils { public static final DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName2("fangjia", "sqmetre_rent", mongoDbUrl) + public static final DBCollection LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia_base", "loudong", mongoDbUrl) + public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", mongoDbUrl) + + public static DSCollection getHouseDataStoreConnection(){ if(house_dataStoreConnection != null) { return house_dataStoreConnection @@ -586,8 +590,8 @@ class CommonDatautils { } - public static def findDistrict(String city, String region, String name, String cat){ - + public static def findDistrictCount(String city, String region, String name, String cat){ + DBObject q = new BasicDBObject(); q.put("city", city); if(region){ @@ -598,7 +602,24 @@ class CommonDatautils { q.put("cat", cat) q.put("status", 0) - + def count = getDataStoreConnection().count(q); + return count; + } + + + public static def findDistrict(String city, String region, String name, String cat){ + + DBObject q = new BasicDBObject(); + q.put("city", city); + if(region){ + q.put("region", region); + } + + q.put("name", name); + q.put("cat", cat) + q.put("status", 0) + + DSCursor cursor = getDataStoreConnection().find(q); if(cursor){ for(def obj : cursor){ @@ -609,6 +630,7 @@ class CommonDatautils { } + public static def findDistrictList(String city, String region, String name, String cat){ DBObject q = new BasicDBObject(); @@ -704,7 +726,7 @@ class CommonDatautils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name","吴中路9弄小区") +// q.put("name","森望芳草绿岸") DBObject keys = new BasicDBObject(); if(fields){ @@ -827,16 +849,24 @@ class CommonDatautils { keys.put("out_pics", 1); keys.put("lat", 1); keys.put("lng", 1); + keys.put("lat2", 1); + keys.put("lng2", 1); keys.put("name_cat", 1); keys.put("finish_building_date", 1) keys.put("mapping_alias", 1) keys.put("sid", 1) - DSCursor cursor = getDataStoreConnection().find(q, keys); - if(cursor){ - for(def obj : cursor){ - return obj; + try{ + DSCursor cursor = getDataStoreConnection().find(q, keys); + if(cursor){ + for(def obj : cursor){ + return obj; + } } + }catch(Exception e){ + println "error_id:"+id + e.printStackTrace() } + return null; } @@ -1053,9 +1083,16 @@ class CommonDatautils { public static def queryDistrictById(def id){ + def dbo = null DBObject q = new BasicDBObject(); - q.put("_id", new ObjectId(id)); - def dbo = getDataStoreConnection().findOne(q); + try{ + q.put("_id", new ObjectId(id.trim().toString())); + dbo = getDataStoreConnection().findOne(q); + }catch(Exception e){ + println id + } + + return dbo; } @@ -2385,6 +2422,7 @@ class CommonDatautils { q.put("block", block); } q.put("cat", cat); + q.put("status",0) def cur = getDataStoreConnection().find(q); return cur.count(); } @@ -3041,18 +3079,26 @@ class CommonDatautils { queryCondition.put("region", region); queryCondition.put("district_name", name); queryCondition.put('weekend', new BasicDBObject('$gte', startDate).append('$lte', endDate)); - DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1)).limit(1) + DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1)) cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) DBObject dbo; + def prices = [] + def index = 0 while(cur.hasNext()) { - dbo = cur.next() - def price = dbo.get("avg_price"); - if(price){ - break; + def obj = cur.next() + if(index == 0){ + dbo = obj } + prices << obj.get("avg_price"); + index++ } - return dbo; + if(prices){ + def avg_price = (int)(prices.sum()/prices.size()) + dbo.avg_price = avg_price + return dbo; + } + return null; } @@ -3184,5 +3230,19 @@ class CommonDatautils { def avg_radios = (int)(radios.sum()/radios.size()) return avg_radios; } + + static def findLoudong(commid){ + def q = new BasicDBObject() + q.put("id_in_seaweed", commid) + def count = LOUDONG_LINE_COLL.count(q) + return count; + } + + static def findHouseLoudong(commid){ + def q = new BasicDBObject() + q.put("district_id", commid) + def count = HOUSE_LOUDONG_COLL.count(q) + return count; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictAroundFacilitiesBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictAroundFacilitiesBatch.groovy index 2a2f5890f..63d7f841f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictAroundFacilitiesBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictAroundFacilitiesBatch.groovy @@ -35,7 +35,7 @@ class DistrictAroundFacilitiesBatch extends BaseBatch { @Override public void process(Object args) { - def cities = ["北京", "上海"] + def cities = ["北京", "上海"]// for (def city : cities) { def ret = CommonDatautils.getAllDistricts("district", city) // def ret = getAllDistrictBaseInfo("上海", '闵行', '万源城', "district"); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy index e8cd3476a..bb7b11f46 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy @@ -6,10 +6,16 @@ import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.jusfoun.JusfounCommon +import com.fangjia.script.util.DateUtil import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection import org.apache.commons.codec.digest.DigestUtils import org.apache.commons.io.FileUtils +import org.bson.types.ObjectId +import java.text.DecimalFormat import java.util.concurrent.TimeUnit /** @@ -19,8 +25,11 @@ import java.util.concurrent.TimeUnit class DistrictPriceExportBatch extends BaseBatch{ def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def lastMonthDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); def charset = "GBK" + public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com") + static main(args) { runBatch(new DistrictPriceExportBatch(), args) } @@ -32,6 +41,9 @@ class DistrictPriceExportBatch extends BaseBatch{ def ret = createDatas("福州") + saveToDB(ret) + + def fileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".csv" def dataCodeFileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".sha512" if(CommonDatautils.isWindow()){ @@ -76,10 +88,10 @@ class DistrictPriceExportBatch extends BaseBatch{ def createDatas(def city){ def districts = CommonDatautils.findAllDistricts(city, "district", null); - - def lastDate = new Date() - def startDate = DateUtils.getFirstDayOfMonth(lastDate, -1) - + def endDateStr = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-10" + def dataStr = DateUtils.add(endDateStr, -30) + def startDate = DateUtils.parseFromTightYYYYMMDD(dataStr) + def lastDate = DateUtils.parseFromTightByPattern(endDateStr, "yyyy-MM-dd") // List list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202111.csv")) // def map2 = [:] // for(def s : list2) { @@ -88,17 +100,34 @@ class DistrictPriceExportBatch extends BaseBatch{ // map2.put(key, row[5]) // } + def lastMonthPriceMap = findFzDistrictStat("福州",lastMonthDate) def result = [] for(def rMap : districts){ + def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate) if(dbo){ + def lastMonthPrice = lastMonthPriceMap.get(rMap.get("_id").toString()) + def curPrice = dbo.get("avg_price") + if(lastMonthPrice && curPrice){ + def markup = (curPrice - lastMonthPrice)*100/lastMonthPrice + if(Math.abs(markup)>3){ + def randMarkup = nextDouble(0, 3) + if(markup < 0){ + markup = randMarkup*(-1) + }else{ + markup = randMarkup + } + curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice) + } + } + def m = [] m << dbo.get("_id").toString()//均价ID m << rMap.get("_id").toString()//楼盘ID m << rMap.city m << rMap.region m << rMap.name - m << dbo.get("avg_price") + m << curPrice // def lastMonthPrice = map2.get(rMap.city + "##" + rMap.region+"##"+rMap.name) // if(lastMonthPrice && dbo.get("avg_price")){ // lastMonthPrice = lastMonthPrice.toInteger() @@ -125,6 +154,19 @@ class DistrictPriceExportBatch extends BaseBatch{ return result } + + def findFzDistrictStat(city,lastMonthDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("month", lastMonthDate) + def dbcursor = FZ_DISTRICT_STAT_COLL.find(q) + def priceMap = [:] + for(def d : dbcursor){ + priceMap.put(d.comm_id, d.avg_price) + } + return priceMap + } + def export(def fileName, def datas){ def headers = ["id","comm_id","city","district","name","avg_price","year_month"] writeCSV(headers, fileName, datas); @@ -152,5 +194,44 @@ class DistrictPriceExportBatch extends BaseBatch{ file.append(System.getProperty("line.separator")); } } + + //随机浮点数 + public static double nextDouble(final double min, final double max) throws Exception { + + DecimalFormat df = new DecimalFormat("#.00"); + if (max < min) { + throw new Exception("min < max"); + } + if (min == max) { + return min; + } + return Double.parseDouble(df.format(min + ((max - min) * new Random().nextDouble()))); + } + + def saveToDB(rets){ + for(def r : rets){ + def q = new BasicDBObject() + q.put("month", curDate.toInteger()) + q.put("comm_id", r[1]) + def obj = FZ_DISTRICT_STAT_COLL.findOne(q) + if(obj){ + def udata = new BasicDBObject() + udata.put("m_date", new Date()) + udata.put("avg_price", r[5]) + FZ_DISTRICT_STAT_COLL.update(q, new BasicDBObject('$set', udata)) + }else{ + def udata = new BasicDBObject() + def o = new BasicDBObject() + o.put("_id", new ObjectId(r[0])) + o.put("comm_id", r[1]) + o.put("city", r[2]) + o.put("region", r[3]) + o.put("name", r[4]) + o.put("avg_price", r[5].toInteger()) + o.put("month", r[6].toInteger()) + FZ_DISTRICT_STAT_COLL.save(o) + } + } + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy index ba848d886..81c5624d1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy @@ -8,7 +8,9 @@ import com.mongodb.Bytes import com.mongodb.DBCollection import com.mongodb.DBCursor import com.mongodb.DBObject +import com.rabbitmq.client.impl.AMQImpl import org.apache.commons.io.FileUtils +import org.bson.types.ObjectId /** * 福州数据结果统计 @@ -17,16 +19,42 @@ class FzDataResultStat { public static final DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName("fangjia", "region_stat", "mongo.db.fangjia.com") + public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com") + static main(args) { def task = new FzDataResultStat() task.priceMarkupRun(); task.regionPriceTrendRun() +//// task.margeExcel() +// task.saveCsvData() + } + + def margeExcel(){ + + def list2= ExcelUtils.read("D:\\数据\\福州\\价格比对_202204_新2.xlsx", 0 ,1) + + def map2 = [:] + for(def s : list2){ + def key = s["A"]+"##"+s["B"]+"##"+s["C"] + map2.put(key, s["D"]) + } + + ExcelUtils.multiThreadingModify("D:\\数据\\福州\\价格比对_202204_新3.xlsx",{row-> + if(row["A"]){ + def key = row["A"]+"##"+row["B"]+"##"+row["C"] + def price = map2.get(key) + if(price){ + row['L'] = price + } + } + }) } def priceMarkupRun(){ def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); List list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv")) + def map1 = [:] for(def s : list1){ def row = s.split("@\\|@") @@ -137,16 +165,13 @@ class FzDataResultStat { queryCondition.put('weekend', new BasicDBObject('$gte', startDate).append('$lte',endDate)); DBCursor cur = REGION_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1)) cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) - def price = 0; + def prices = []; while(cur.hasNext()) { DBObject dbo = cur.next() - price = dbo.get("avg_price"); - if(price){ - break; - } + prices << dbo.get("avg_price"); } - - return price; + def avg_price = (int)(prices.sum()/prices.size()) + return avg_price; } @@ -165,4 +190,45 @@ class FzDataResultStat { weekends = weekends.reverse(); return weekends; } + + + def saveCsvData(){ + def files = getAllFiles("C:\\Users\\gaojuan\\福州数据(1)") + if(files){ + for(def filepath : files){ + List list1= FileUtils.readLines(filepath) + println filepath + for(def s : list1){ + def row = s.split("@\\|@") + def o = new BasicDBObject() + o.put("_id", new ObjectId(row[0])) + o.put("comm_id", row[1]) + o.put("city", row[2]) + o.put("region", row[3]) + o.put("name", row[4]) + o.put("avg_price", row[5].toInteger()) + o.put("month", row[6].toInteger()) + FZ_DISTRICT_STAT_COLL.save(o) + + } + } + } + } + + def static getAllFiles(def path) { + + File file = new File(path); + File[] tempList = file.listFiles(); + + def files = []; + for (int i = 0; i < tempList.length; i++) { + if (tempList[i].isFile()) { + files << tempList[i]; + } + if (tempList[i].isDirectory()) { + System.out.println("文件夹:" + tempList[i]); + } + } + return files; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index 5a6b21aac..7e5074c7c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202203.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202204.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/TradeExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/TradeExportBatch.groovy index 244497f17..cd6252fb6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/TradeExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/TradeExportBatch.groovy @@ -112,22 +112,28 @@ class TradeExportBatch extends BaseBatch{ // q.put("trade_date", new BasicDBObject(QueryOperators.LT, endDate)) def cursor = DEAL_PRICE.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(1000); def rets = [] + def keys = [] if (cursor != null) { while (cursor.hasNext()) { DBObject dbo = cursor.next(); def district = districtMap.get(dbo.city + "_" + dbo.region + "_" + dbo.district_name); if(district){ - def ret = []; - ret << dbo._id.toString() - ret << district._id.toString() - ret << dbo.city - ret << dbo.region - ret << dbo.district_name - ret << dbo.avg_price - ret << dbo.area - ret << DateUtils.formatDateByPattern(dbo.trade_date, "yyyy-MM-dd") - ret << dbo.total_price - rets << ret; + def tradeDateStr = DateUtils.formatDateByPattern(dbo.trade_date, "yyyy-MM-dd") + def key = district._id.toString() +"##" + dbo.avg_price +"##"+dbo.area+"##"+tradeDateStr + if(!keys.contains(key)){ + keys << key + def ret = []; + ret << dbo._id.toString() + ret << district._id.toString() + ret << dbo.city + ret << dbo.region + ret << dbo.district_name + ret << dbo.avg_price + ret << dbo.area + ret << tradeDateStr + ret << dbo.total_price + rets << ret; + } } } println "成交总计:"+rets.size() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHousePriceStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHousePriceStat.groovy index 4628a207b..d5123668b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHousePriceStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHousePriceStat.groovy @@ -35,7 +35,8 @@ class EsHousePriceStat extends BaseBatch{ def run(){ def cityMap = CommonDatautils.getAllCityRegions() cityMap.each{city,regions-> - findPriceByCity(city, regions); + + findPriceByCity(city, regions); } } @@ -64,9 +65,9 @@ class EsHousePriceStat extends BaseBatch{ q.put("region" , r.region) q.put("month", r.month) def mom = null - if(lastObj){ - mom = (r.price -lastObj.avg_price)/lastObj.avg_price - mom = DataHandleUtils.getDoubleRound(4, mom) + if(lastObj && lastObj.avg_price && r.price){ + mom = (r.price -lastObj.avg_price)/lastObj.avg_price + mom = DataHandleUtils.getDoubleRound(4, mom) } if(obj){ if(obj.avg_price != r.price){ @@ -103,7 +104,7 @@ class EsHousePriceStat extends BaseBatch{ def findEsAvgPrice(city, region){ def detail = [:] - detail.put("date_histogram",[ "field": "publish_date","interval": "1M","time_zone": "Asia/Shanghai","min_doc_count": 1]) + detail.put("date_histogram",[ "field": "publish_date","interval": "1M","time_zone": "Asia/Shanghai","min_doc_count": 0]) detail.put("aggs",[ "1": ["moving_avg": ["buckets_path": "1-metric"]],"1-metric": ["avg": ["field": "avg_price"]]]) def must = [] @@ -143,15 +144,18 @@ class EsHousePriceStat extends BaseBatch{ if(!priceValue){ priceValue = row['1-metric'] } - if(priceValue){ + if(priceValue && priceValue.value){ price = priceValue.value.toInteger() } result.price = price; def date = row.key_as_string.replace("-","").substring(0, 6) def curMonth = DateUtils.formatDateByPattern(new Date(),"yyyyMM") if(date!=curMonth){ - result.month = date.toInteger() - rets << result + if(date.toInteger()>200001 && result.price){ + result.month = date.toInteger() + rets << result + } + } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy index 77239e3ea..e4b621c99 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy @@ -164,6 +164,7 @@ class OpenApiUtils { if(StringUtils.isEmpty(city) || StringUtils.isEmpty(address)){ return null; } + address = address.trim().replace(" ","") String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&types=&city=" + city + "&output=json&offset=20&page=1&key=2af0b8533415a612b98d59bb93d664b9&extensions=all" List poiList = null; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCityPriceStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCityPriceStat.groovy index c60a811b5..19709dd22 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCityPriceStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCityPriceStat.groovy @@ -18,44 +18,62 @@ class CcbCityPriceStat { task.run() } def run(){ - def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + def citys = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","临汾","聊城","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","齐齐哈尔","眉山","郴州","营口","蚌埠","赤峰","大庆","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","朔州","随州","大理","通化","楚雄","巴彦淖尔","乌兰察布","松原","景德镇","新余","辽阳","阜新","黔南","酒泉","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","钦州","内江","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","鹤岗","萍乡","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","湘西","昭通","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"] def rets = [] + def startDate = DateUtils.parseFromTightByPattern("2022-02-01","yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-06-01","yyyy-MM-dd") + for(def city : citys){ - def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias"]) - def priceMap = CommonDatautils.queryDistrictPrice(city, DateUtils.parseFromTightYYYYMMDD("2021-05-30")) - def cityRets = [] - for(def dist : districts){ - def tradeAvgPrice = findTrades(dist, "2021-02-01", "2021-06-01") - def weekendPrice = priceMap.get(city +"##"+dist.region+"##"+dist.name) - if(tradeAvgPrice && weekendPrice){ - def row = [city, dist.region, dist.name, weekendPrice, tradeAvgPrice.toInteger()] - cityRets << row - println row.toString() + def matchTotalCount = findMatchTradeCount(city, startDate, endDate) + println city +" "+matchTotalCount + if(matchTotalCount){ + def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias"]) + def priceMap = CommonDatautils.queryDistrictPrice(city, DateUtils.parseFromTightYYYYMMDD("2022-05-29")) + def cityRets = [] + for(def dist : districts){ + def tradeAvgPrice = findTrades(dist, "2022-02-01", "2022-06-01") + def weekendPrice = priceMap.get(city +"##"+dist.region+"##"+dist.name) + if(tradeAvgPrice && weekendPrice){ + def row = [city, dist.region, dist.name, weekendPrice, tradeAvgPrice.toInteger()] + cityRets << row + println row.toString() + } } + print city +" "+ cityRets.size() + rets.addAll(cityRets) } - print city +" "+ cityRets.size() - rets.addAll(cityRets) + } println "总量:"+rets.size() - ExcelUtils.write("d:\\数据\\92个城市价格数据.xlsx", rets, "所有") + ExcelUtils.write("d:\\数据\\234个城市价格数据.xlsx", rets, "所有") + } + + + def findMatchTradeCount(city, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + q.put("match_id", new BasicDBObject('$exists', true)) + def count = DEAL_PRICE_COLL.count(q) + return count } def findTrades(dist, startDate, endDate){ def q = new BasicDBObject() - q.put("city", dist.city) - q.put("region", dist.region) - - if(dist.alias){ - if(!dist.alias.contains(dist.name)){ - dist.alias << dist.name - } - q.put("district_name", new BasicDBObject('$in', dist.alias)) - }else{ - q.put("district_name", dist.name) - } - +// q.put("city", dist.city) +// q.put("region", dist.region) +// +// if(dist.alias){ +// if(!dist.alias.contains(dist.name)){ +// dist.alias << dist.name +// } +// q.put("district_name", new BasicDBObject('$in', dist.alias)) +// }else{ +// q.put("district_name", dist.name) +// } + q.put("match_id", dist._id.toString()) q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate))) def dbcorsor = DEAL_PRICE_COLL.find(q) def avg_price = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy index 1dabf3ada..66ff6a8f3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy @@ -199,18 +199,18 @@ class CcbCodeUtils { static def getBuildStructureCode(def buildStructure){ if(buildStructure){ if(buildStructure.contains("钢结构")){ - return "001"; + return "3"; }else if(buildStructure.contains("钢、钢筋混凝土结构")){ - return "002"; + return "1"; }else if(buildStructure.contains("钢筋混凝土结构")){ - return "003"; + return "1"; }else if(buildStructure.contains("砖木结构")){ - return "004"; + return "10"; }else if(buildStructure.contains("混合结构")){ - return "005"; + return "2"; } } - return "006"; + return "11"; } static def getDistHouseTypeCode(def estateType){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy index 3e4981856..23513b80b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy @@ -50,4 +50,32 @@ class FileUtils { } out.close(); } + + static def createTxtFile(ret, fileName, charset, isAppend){ + File file = new File(fileName); + if (!file.getParentFile().exists()) { + boolean result = file.getParentFile().mkdirs(); + if (!result) { + System.out.println("创建目录失败"); + } + } + + OutputStream out = new FileOutputStream(fileName, isAppend); + def sb = new StringBuffer(); + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + def index = 0 + for(def row : ret){ + if(row){ + index++ + sb.append(row) + sb.append("\n"); + } + } + if(sb.length()>0){ + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + } + out.close(); + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy new file mode 100644 index 000000000..d55350598 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy @@ -0,0 +1,257 @@ +package com.fangjia.script.gj.crawl + +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil +import com.fangjia.script.thirdPriceCrawl.Utils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.rabbitmq.client.impl.AMQImpl +import net.sf.json.JSONObject +import org.jsoup.Jsoup +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.ChromeDriver + +class AnjukeAppCrawl { + + public static final SHORT_DATE = Utils.getLastMonth(); + def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); + public static def SOURCE = "安居客"; + DBCollection detail = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "app_anjuke_detail") + + static main(args) { + def task = new AnjukeAppCrawl(); +// task.run(); +// task.run2() + + task.addThirdPrice() + } + + def run2(){ + def url = "https://m.anjuke.com/sh/community/" + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + WebDriver driver = new ChromeDriver() + def links = getRegion(url, driver) + for(def link : links){ + def doc = getDocument(url, driver) + def str = doc.toString().substring(doc.toString().indexOf("")) + if(str){ + str = str.substring(str.indexOf("} ("), str.indexOf("));")) + def arrs = str.split(",") + pritln arrs.size() + } + + } + } + + def getDocument(url, driver){ + def stop = false + def error = 0 + def doc = null + while(!stop){ + try{ + driver.get(url) + doc = Jsoup.parse(driver.getPageSource()); + if(doc && !doc.toString().contains("访问验证-安居客") && !doc.toString().contains("502 Bad Gateway") && !doc.toString().contains("登录")){ + stop = true; + }else{ + error++ + driver.manage().deleteAllCookies() + println "[${error}] error: "+url + Thread.sleep(5000) + } + }catch(Exception e){ + error++ + } + if(error>=20){ + stop = true; + } + } + return doc + } + + def getRegion(url, driver){ + //def doc = getDocument(url, driver) + def header = [:] + header.put("accept","application/json, text/plain, */*") + header.put("accept-encoding","gzip, deflate, br") + header.put("accept-language","zh-CN,zh;q=0.9") + header.put("cookie","aQQ_ajkguid=E9796514-38C0-44D2-8CA3-1C58C3928B29; sessid=765EEB9D-E7A2-4B2C-8151-CA9CD939AB02; id58=CpQB22J8sqMA9stuJfeAAg==; twe=2; 58tj_uuid=bf6eaff8-851c-471a-a914-853242878ddc; als=0; seo_source_type=1; _ga=GA1.2.215255179.1652344695; isp=true; ajk-appVersion=; obtain_by=2; fzq_js_anjuke_ershoufang_m=c2c12b2d1b221734fa20ef752cd6bdc0_1652509223606_27; fzq_h=ef0186eeaba082ecd2a47ede7c7e3e64_1652533612190_a34c026e9c1842a3b35bdd37f3d2aa0b_2015773878; init_refer=https%253A%252F%252Fm.anjuke.com%252Fcq%252Fcommunity%252F; new_uv=8; wmda_uuid=b59ce2efbba070ecfa8fe93d79f30dd4; wmda_new_uuid=1; wmda_session_id_6145577459763=1652535525740-dae51249-3379-d4e7; wmda_visited_projects=%3B6145577459763; new_session=0; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; xxzl_deviceid=q9XQbdA9hhkHJTj+RgVpwc7jrBzVJp9J1uTlp4hDtRiXtA615K4HS4R0pJ1dgrBa; ctid=20; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; fzq_js_anjuke_xiaoqu_m=c80eeca4a3301dbf27b3c79b9f18fa3b_1652536277223_27; xzuid=81992155-5990-4a2f-8842-32bcbfbb1117") + header.put("referer","https://m.anjuke.com/fz/") + header.put("sec-fetch-dest","empty") + header.put("sec-fetch-mode","cors") + header.put("sec-fetch-site","same-origin") + header.put("sw8","1-ODA0ZDA3N2ZmMDA1NDgyYmExNTFlODhlZWIyNmFkMzk=-N2UwZDkxMGU2ZTdjNDdkZmJlZDBkMjUzMmNkODQ2NWU=-0-MTE3-aHR0cHM6Ly9tLmFuanVrZS5jb20=-L2VzZi1hamF4L2NvbW11bml0eS9saXN0-bS5hbmp1a2UuY29t-1") + header.put("sw8-correlation","c291cmNl:YmVpZG91") + header.put("user-agent","Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1") + + def doc = ABuYunUtil.doGet(url, null, header, "utf-8") + def links = [] + if(doc){ + def eles = JQueryUtils.select(".column2 a", doc) + for(def e : eles){ + def href = e.attr("href") + links << href + } + } + return links; + } + + + + + def run(){ + //上海:11,北京:14 + //"仓山":4071, + // def map = ["晋安":4070,"鼓楼":4068,"闽侯":4073,"台江":4069,"福清":16378,"长乐":15880,"马尾":4072,"连江":16380,"平潭":16387,"罗源":16382,"永泰":16392,"闽清":16384] + // def regionMap = ["浦东":7,"闵行":11, "松江":15,"宝山":13,"嘉定":14,"徐汇":5,"青浦":19,"静安":2,"普陀":10,"杨浦":9,"奉贤":17,"黄浦":4,"虹口":8,"长宁":6,"金山":18,"崇明":20] + def regionMap = [ "朝阳":642,"海淀":601,"昌平":843,"丰台":753,"大兴":865,"通州":818,"房山":802,"顺义":837,"西城":704,"东城":690,"密云":856,"石景山":787,"怀柔":858,"门头沟":815,"延庆":860,"平谷":863,"北京周边":2678] + def cityMap = ["北京":regionMap] + def cityId = 14 + regionMap.each{region, area_id-> + for(def i=1;i<40;i++){ + def stop = false; + def all_stop = false + def error_count=0 + while(!stop){//渝北:2359 南岸:2368 沙坪坝:2380 + //城口 + def flagMap = getInfo(stop, i, area_id, cityId, "上海") + stop = flagMap.stop + all_stop = flagMap.all_stop + error_count++ + if(error_count>=20){ + stop = true + all_stop = true + } + } + if(all_stop){ + break; + } + } + } + } + + def getInfo(def stop, def i, def area_id, def cityId, def city){ + + def map = [:] + def url = "https://m.anjuke.com/esf-ajax/community/list?city_id=${cityId}&page_size=100&page=${i}&area_id=${area_id}&is_tw=1" + + def header = [:] + header.put("accept","application/json, text/plain, */*") + header.put("accept-encoding","gzip, deflate, br") + header.put("accept-language","zh-CN,zh;q=0.9") + header.put("cookie","aQQ_ajkguid=E9796514-38C0-44D2-8CA3-1C58C3928B29; sessid=765EEB9D-E7A2-4B2C-8151-CA9CD939AB02; id58=CpQB22J8sqMA9stuJfeAAg==; twe=2; 58tj_uuid=bf6eaff8-851c-471a-a914-853242878ddc; als=0; seo_source_type=1; _ga=GA1.2.215255179.1652344695; isp=true; ajk-appVersion=; obtain_by=2; fzq_js_anjuke_ershoufang_m=c2c12b2d1b221734fa20ef752cd6bdc0_1652509223606_27; fzq_h=ef0186eeaba082ecd2a47ede7c7e3e64_1652533612190_a34c026e9c1842a3b35bdd37f3d2aa0b_2015773878; init_refer=https%253A%252F%252Fm.anjuke.com%252Fcq%252Fcommunity%252F; new_uv=8; wmda_uuid=b59ce2efbba070ecfa8fe93d79f30dd4; wmda_new_uuid=1; wmda_session_id_6145577459763=1652535525740-dae51249-3379-d4e7; wmda_visited_projects=%3B6145577459763; new_session=0; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; xxzl_deviceid=q9XQbdA9hhkHJTj+RgVpwc7jrBzVJp9J1uTlp4hDtRiXtA615K4HS4R0pJ1dgrBa; ctid=20; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; fzq_js_anjuke_xiaoqu_m=c80eeca4a3301dbf27b3c79b9f18fa3b_1652536277223_27; xzuid=81992155-5990-4a2f-8842-32bcbfbb1117") + header.put("referer","https://m.anjuke.com/sh/community/") + header.put("sec-fetch-dest","empty") + header.put("sec-fetch-mode","cors") + header.put("sec-fetch-site","same-origin") + header.put("sw8","1-ODA0ZDA3N2ZmMDA1NDgyYmExNTFlODhlZWIyNmFkMzk=-N2UwZDkxMGU2ZTdjNDdkZmJlZDBkMjUzMmNkODQ2NWU=-0-MTE3-aHR0cHM6Ly9tLmFuanVrZS5jb20=-L2VzZi1hamF4L2NvbW11bml0eS9saXN0-bS5hbmp1a2UuY29t-1") + header.put("sw8-correlation","c291cmNl:YmVpZG91") + header.put("user-agent","Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1") + + def ret = ABuYunUtil.doGet(url, null, null, "utf-8") + def all_stop = false; + def error_count = 0 + if (ret) { + JSONObject jsonObj = JSONObject.fromObject(ret); + if(jsonObj && jsonObj.status=="ok" ){ + def data = jsonObj.data + if(data && data.total.toInteger()>0){ + def communities = data.communities + + println "【${i}】 "+communities.size() +" "+url + if(communities.size()>0){ + save(city, communities) + Thread.sleep(10000) + stop = true; + }else{ + stop = true; + println url +" 返回0" + all_stop = true; + } + + }else{ + stop = false; + error_count ++ + println url +" 无数据" + } + + + }else{ + stop = false; + Thread.sleep(5000) + } + } + + map.stop = stop + map.all_stop = all_stop; + return map; + } + + def save(city, def ret){ + def index = 0 + for(def o : ret){ + index++ + def obj = findIsExist(o.jumpAction) + if(!obj){ + def udate = new BasicDBObject() + o.each{k,v-> + udate.put(k,v) + } + udate.put("city", city) + udate.put("c_date", new Date()) + detail.save(udate) + println index+" [保存]"+o.jumpAction + }else{ + println index+" [已存在]"+o.jumpAction + } + + } + } + + def findIsExist(def url){ + def q = new BasicDBObject() + q.put("jumpAction",url) + return detail.findOne(q); + } + + def addThirdPrice(){ + def q = new BasicDBObject() + def dbcursor = detail.find(q) + def index = 0 + for(def d : dbcursor){ + index++ + def city = d.base.cityName + def region = d.base.areaName + def name = d.base.name + def obj = findByParams(city, region, name); + if(!obj){ + def data = new BasicDBObject() + data.put("district_id",d.base.id) + data.put("city", city) + data.put("region", region ) + data.put("name", name) + data.put("address",d.base.address) + data.put("avg_price", d.priceInfo.price) + data.put("source", SOURCE) + data.put("s_date", SHORT_DATE) + data.put("c_date", d.c_date) + thirdPartyColl.save(data) + println index +" "+data + } + } + } + + + def findByParams(city, region, name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put("source", SOURCE) + q.put("s_date", SHORT_DATE) + return thirdPartyColl.findOne(q); + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDetailCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDetailCrawl.groovy index d4f3001b3..ab7aa4278 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDetailCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDetailCrawl.groovy @@ -11,9 +11,15 @@ import org.openqa.selenium.By import org.openqa.selenium.WebDriver import org.openqa.selenium.WebElement import org.openqa.selenium.chrome.ChromeDriver +import org.openqa.selenium.remote.DesiredCapabilities +import org.openqa.selenium.remote.RemoteWebDriver class AnjukeDetailCrawl { + DBCollection detailColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_detail") + + DBCollection urlColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "detail_url") + static main(args) { def task = new AnjukeDetailCrawl(); task.run(); @@ -21,9 +27,15 @@ class AnjukeDetailCrawl { def run(){ System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); System.setProperty("http.nonProxyHosts", "localhost"); - def url = "https://chongqing.anjuke.com/community/?from=navigation" + def url = "https://chongqing.anjuke.com/" WebDriver driver = new ChromeDriver() +// DesiredCapabilities capabilities = DesiredCapabilities.chrome(); +// URL urlInstance = new URL("http://192.168.0.100:30025"); //指定URL +// WebDriver driver = new RemoteWebDriver(urlInstance, capabilities); + + driver.manage().deleteAllCookies() def regions = getRegions(driver, url) + println "区域链接: "+regions.size() getAllRegions(driver, regions) } @@ -31,44 +43,88 @@ class AnjukeDetailCrawl { def getAllRegions(driver, regions){ for(def url : regions){ - driver.get(url) - def doc = Jsoup.parse(driver.getPageSource()); + def doc = getDocument(url, driver) def total = getTotal(doc) if(total) { if (total > 1250) { - + def links = getMoreLinks(url, driver) + println url +" "+links.size() + saveDetailLinks(links) +// parse(links,driver) } else { def links = getAllLinks(url, driver) - parse(links,driver) + println url +" "+links.size() + saveDetailLinks(links) +// parse(links,driver) } } } } + def saveDetailLinks(links){ + for(def url : links){ + def data = new BasicDBObject() + data.put("city","重庆") + data.put("url", url) + data.put("source", "安居客") + data.put("c_date", new Date()) + urlColl.save(data) + } + } + + def getMoreLinks(url, driver){ + def doc = Jsoup.parse(driver.getPageSource()); + def pricelinks = JQueryUtils.select(".filter-wrap:contains(均价) .line-item a", doc) + def links = [] + if(pricelinks){ + for(def link : pricelinks){ + if(link.text()!="不限"){ + def urlLink = link.attr("href") + getDocument(urlLink, driver) +// driver.get(urlLink) + links.addAll(getAllLinks(urlLink, driver)) + println urlLink +" "+links.size() + } + } + } + return links; + } + def getAllLinks(url, driver){ def stop = false def urls = [] - while(!stop) { - def doc = Jsoup.parse(driver.getPageSource()); + def doc = Jsoup.parse(driver.getPageSource()); + def total = getTotal(doc) + def page = getPage(total, 25) + def pages = [] + for(def i=1;i<=page;i++){ + pages << url.substring(0, url.size()-1)+"-p"+i+"/" + } + for(def pageUrl : pages){ + doc = getDocument(pageUrl, driver) def details = JQueryUtils.select(".list-cell a.li-row", doc) - for (def d : details) { - urls << d.attr("href") - } - def nextEle = JQueryUtils.select(".next", doc) - if (nextEle) { - driver.findElement(By.cssSelector(".next")).click() - }else{ - stop = true - } - if(urls.size()>20){ - stop = true + def link = d.attr("href") + if(!urls.contains(link)){ + urls << d.attr("href") + } } } return urls } + def getPage(def total, def pageSize){ + if(total && total.toInteger()>0){ + def page = total.toInteger()/pageSize; + if(page>(int)page){ + page = (int)page + 1 + return page + } + return (int)page; + } + return null + } def getTotal(doc){ def totalText = JQueryUtils.selectText(".total-info", doc) @@ -82,9 +138,19 @@ class AnjukeDetailCrawl { } def getRegions(driver, url){ +// def doc = getDocument(url, driver) driver.get(url) - Thread.sleep(3000) def doc = Jsoup.parse(driver.getPageSource()); + def els = JQueryUtils.select("a:contains(小区)", doc) + if(els){ + def href = els[0].attr("href") + doc = Jsoup.parse(driver.getPageSource()); + // driver.findElement(By.xpath("//a[contains(text(), '小区')]")).click() +// driver.findElement(By.linkText("小区")).click() + doc = getDocument(href, driver) + } + + // def loginIfm = JQueryUtils.select("#iframeLoginIfm", doc) // if(loginIfm){ // def href = loginIfm[0].attr("src") @@ -107,23 +173,35 @@ class AnjukeDetailCrawl { def regionEles = JQueryUtils.select(".region-parents li a", doc) def urls = [] for(def e : regionEles){ - urls << e.attr("href") + if(e.text()!="不限"){ + urls << e.attr("href") + } } return urls; } + def getNontCheckPage(url, driver, doc){ + def stop = false; + while(!stop){ + if(doc){ + + } + } + return doc; + } + def parse(def links, driver){ def rets = [] for(def link : links){ def district = parsing(link, driver) if(district){ - rets << district + // rets << district - if (rets.size() % 100 == 0) { - def add= saveData(rets) - println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") - rets = [] - } + // if (rets.size() % 100 == 0) { + saveObj(district) + println district.name+" "+district.url + " "+DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + // rets = [] + // } } } if (rets.size()> 0) { @@ -136,33 +214,61 @@ class AnjukeDetailCrawl { def saveData(def results){ def add = 0 for(def ret : results){ - def q = new BasicDBObject() - q.put("url", ret.url) - def dbo = detailColl.findOne(q); - - DBObject obj = new BasicDBObject(); - ret.each{k,v-> - obj.put(k,v) - } - - if(dbo){ - obj.m_date = new Date() - detailColl.update(q, new BasicDBObject('$set', obj)) - }else{ - add++ - obj.c_date = new Date() - obj.m_date = new Date() - try{ - detailColl.save(obj) - }catch(Exception e){ - println obj.url - } - } + saveObj(ret) } - return add } + def saveObj(def ret){ + def q = new BasicDBObject() + q.put("url", ret.url) + def dbo = detailColl.findOne(q); + + DBObject obj = new BasicDBObject(); + ret.each{k,v-> + obj.put(k,v) + } + + if(dbo){ + obj.m_date = new Date() + detailColl.update(q, new BasicDBObject('$set', obj)) + }else{ + + obj.c_date = new Date() + obj.m_date = new Date() + try{ + detailColl.save(obj) + }catch(Exception e){ + println obj.url + } + } + } + def getDocument(url, driver){ + def stop = false + def error = 0 + def doc = null + while(!stop){ + try{ + driver.get(url) + doc = Jsoup.parse(driver.getPageSource()); + if(doc && !doc.toString().contains("访问验证-安居客") && !doc.toString().contains("502 Bad Gateway") && !doc.toString().contains("登录")){ + stop = true; + }else{ + error++ + driver.manage().deleteAllCookies() + println "[${error}] error: "+url + Thread.sleep(5000) + } + }catch(Exception e){ + error++ + } + if(error>=20){ + stop = true; + } + } + return doc + } + def parsing(url, driver){ @@ -191,10 +297,9 @@ class AnjukeDetailCrawl { def infoDiv = JQueryUtils.select(".info-list", doc) district.estate_type = getValue(infoDiv.select(".column-2:contains(物业类型) .value").text()) - district.estate_charge = getValue(infoDiv.select(".column-2:contains(物业费) .value").text()) - district.estate_company = getValue(infoDiv.select(".column-1:contains(物业公司) .value").text()) - district.developer = getValue(infoDiv.select(".column-1:contains(开发商) .value").text()) - + district.quanshu_type = getValue(infoDiv.select(".column-2:contains(权属类别) .value").text()) + district.finish_date = getValue(infoDiv.select(".column-2:contains(竣工时间) .value").text()) + district.property_life = getValue(infoDiv.select(".column-2:contains(产权年限) .value").text()) def houseCountStr = infoDiv.select(".column-2:contains(总户数) .value").text() if (houseCountStr) { houseCountStr = houseCountStr.replace("户", "") @@ -202,9 +307,13 @@ class AnjukeDetailCrawl { district.house_count = houseCountStr.toInteger() } } - - district.finish_date = getValue(infoDiv.select(".column-2:contains(竣工时间) .value").text()) district.build_area = getValue(infoDiv.select(".column-2:contains(总建面积) .value").text()) + district.volumetric_rate = getValue(infoDiv.select(".column-2:contains(容积率) .value").text()) + district.greening_rate = getValue(infoDiv.select(".column-2:contains(绿化率) .value").text()) + + district.build_type = getValue(infoDiv.select(".column-2:contains(建筑类型) .value").text()) + district.business = getValue(infoDiv.select(".column-2:contains(所属商圈) .value").text()) + def carSiteStr = getValue(infoDiv.select(".column-2:contains(停车位) .value").text()) if(carSiteStr){ if(carSiteStr.contains("(")){ @@ -214,16 +323,19 @@ class AnjukeDetailCrawl { district.car_site = carSiteStr } } - district.volumetric_rate = getValue(infoDiv.select(".column-2:contains(容积率) .value").text()) - district.greening_rate = getValue(infoDiv.select(".column-2:contains(绿化率:) .value").text()) - district.build_type = getValue(infoDiv.select(".column-2:contains(建筑类型) .value").text()) - district.property_life = getValue(infoDiv.select(".column-2:contains(产权年限) .value").text()) - district.car_site_charge = getValue(infoDiv.select(".column-1:contains(停车费) .value").text()) + district.car_site_str = infoDiv.select(".column-2:contains(停车位) .value").text() + district.estate_charge = getValue(infoDiv.select(".column-2:contains(物业费) .value").text()) + + district.stop_charge = getValue(infoDiv.select(".column-1:contains(停车费) .value").text()) + district.car_site_charge = getValue(infoDiv.select(".column-1:contains(车位管理费) .value").text()) + + district.estate_company = getValue(infoDiv.select(".column-1:contains(物业公司) .value").text()) + district.developer = getValue(infoDiv.select(".column-1:contains(开发商) .value").text()) district.url = url district.c_date = new Date() district.m_date = new Date() + district.crawl_type = "chrome" district.root = "anjuke.com" - stop = true; }else{ Thread.sleep(3000) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy new file mode 100644 index 000000000..f990711be --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy @@ -0,0 +1,89 @@ +package com.fangjia.script.gj.crawl + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient + +class AnjukeDistrictMatch { + + static main(args) { + def task = new AnjukeDistrictMatch(); + task.run(); +// task.run1() + } + + def run1(){ + def regionPointsMap = findRegionRange("重庆") + def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region)) + } + def run(){ + + + + def filePath = "D:\\数据\\重庆农商行\\安居客\\重庆小区.xlsx" + ExcelUtils.multiThreadingModify(filePath,{row-> + if(row['A'] && !row['J']){ + def city = row['B'] + def region = row['C'] + def name = row['D'] + def address = row['E'] +// def flag = "别名匹配" + + +// def dist = matchDist(city, null, name) +// if(!row['J'] && name.contains("(")){ +// name = name.replace("(${region})","").replace("(","").replace(")","") +// flag = "去括号匹配" +// dist = matchDist(city, region, name) +// } +// if(!dist){ +// dist = CommonDatautils.findSeaweedByMappingAlias(city, region, name,"district") +// flag = "映射别名匹配" +// } +// if(!dist) { +// def rets = CommonDatautils.findDistrictByAlias(city, region, name, "district") +// if(rets && rets.size()==1){ +// dist = rets[0] +// } +// } + +// +// if(!dist){ +// def dists = CommonDatautils.findDistrictByAddress(city, region, address,"district") +// flag = "地址匹配" +// if(dists.size()>1){ +// println address +" 地址相同:"+dists.size() +// }else{ +// if(dists.size()==1){ +// dist = dists[0] +// } +// } +// } + +// if(dist){ +// row['J'] = flag +// row['K'] = dist._id.toString() +// row['L'] = dist.region +// row['M'] = dist.name +// row['N'] = dist.address +// } + + def jsonObject = OpenApiClient.addressMatch(city, address, "property", false,region,name) + if(jsonObject != null && jsonObject.precise!=-1) { +// row['J'] = "匹配接口" +// row['K'] = jsonObject.get("id") +// row['L'] = jsonObject.get("district") +// row['M'] = jsonObject.get("name") +// row['N'] = jsonObject.get("address") + row['M'] = jsonObject.get("name") + } + } + }) + } + + def matchDist(city, region, name){ + def dist = CommonDatautils.findDistrict(city, region, name, "district") + return dist; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy new file mode 100644 index 000000000..1773bdd8f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy @@ -0,0 +1,189 @@ +package com.fangjia.script.gj.crawl + +import com.alibaba.fastjson.JSON +import com.alibaba.fastjson.JSONObject +import com.fangjia.core.RegionData +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.gj.EsUtils +import com.fangjia.script.temp.shenjian.util.MessageUtils +import com.fangjia.script.util.CommonUtils +import org.jsoup.Jsoup +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.ChromeDriver + +class AnjukeHouseCrawl { + def targetUrl = "http://192.168.0.206:9200/amap_poi/_doc" + static main(args) { + def task = new AnjukeHouseCrawl(); + task.run(); + } + + def run(){ + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + WebDriver driver = new ChromeDriver() + for(def i=24;i<50;i++){ + def url = "https://chengde.anjuke.com/sale/shuangqiao/p${i}/?from=navigation" + def doc = getDocument(url, driver) + def eles = JQueryUtils.select(".list-main .list-left .list .property a", doc) + println "页数:"+i+" 记录数:"+eles.size() + if(eles){ + parser(eles, driver) + } + } + } + + def parser(eles, driver){ + def index = 0 + for(def e : eles){ + index ++ + def href = e.attr("href") + def doc = getDocument(href, driver) +// parserHtml(doc) + def newUrl = href.substring(0, href.indexOf("?")) + sendMq(newUrl, doc) + + println "[ "+index +"] "+ newUrl + } + } + + def sendMq(href, doc){ + JSON json = new JSONObject() + json.put("project", "sell_anjuke") + json.put("url", href) + + JSON result = new JSONObject() + result.put("content", doc.toString()) + json.put("result", result) + + byte[] packedMsg = MessageUtils.pack(json) + CommonUtils.sendMessage("result2parse", packedMsg) + } + + def parserHtml(doc) { + String city = null + def result = [:] + result.siteCode = "安居客" + if (doc) { + result.title = JQueryUtils.selectText([".banner-title h1"], doc) + def heads = JQueryUtils.select(".crumbs a", doc) + + if (heads && heads.size() >= 5) { + result.city = heads[0].text().replace("房产网", "") + def orgRegion = heads[2].text().replace("二手房", "") + result.region = RegionData.formatRegion(result.city, orgRegion.trim()) + result.block = heads[3].text().replace("二手房", "") + result.district_name = heads[4].text() + } else if (heads && heads.size() >= 3) { + result.city = heads[0].text().replace("房产网", "") + result.district_name = heads[2].text() + def eles = JQueryUtils.select("li:contains(所在位置:) a", doc) + if (eles && eles.size() > 0) { + def orgRegion = eles[0].text() + result.region = RegionData.formatRegion(result.city, orgRegion.trim()) + } + } + + if (!result.district_name) { + println " [未解析出(城市-区域-名称)信息]" + } + println result + + + def priceStr = JQueryUtils.selectText(".maininfo-price-num", doc) + if (priceStr) { + if (priceStr.isNumber()) { + result.total_price = priceStr.toDouble() * 10000 + } + } + def roomInfoStr = JQueryUtils.selectText(".maininfo-model-item-1 .maininfo-model-strong", doc) + if (roomInfoStr) { + Integer room = DataCrawlUtils.parseRoom(roomInfoStr) + Integer hall = DataCrawlUtils.parseHall(roomInfoStr) + Integer toilet = DataCrawlUtils.parseToilet(roomInfoStr) + + if (room) { + result.room = room + } + if (hall) { + result.hall = hall + } + if (toilet) { + result.toilet = toilet + } + } + + def areaStr = JQueryUtils.selectText(".maininfo-model-item-2 .maininfo-model-strong", doc).replace("㎡", "") + if (areaStr) { + result.area = areaStr.toDouble() + } + def directionStr = JQueryUtils.selectText(".maininfo-model-item-3 i", doc) + result.direction = DataCrawlUtils.parseDirection(directionStr) + + def buildDateStr = JQueryUtils.selectText(".maininfo-model-item-3 .maininfo-model-weak", doc) + if (buildDateStr && buildDateStr.contains("/")) { + def arrs = buildDateStr.split("\\/") + if (arrs && arrs.size() == 2) { + result.build_date = arrs[0] + result.estate_type = arrs[1] + } + } + + result.floor_info = JQueryUtils.selectText(".maininfo-model-item-1 .maininfo-model-weak", doc) + if (result.floor_info && result.floor_info.contains("共")) { + def height = result.floor_info + if (result.floor_info.contains("(")) { + height = height.substring(result.floor_info.indexOf("(") + 1, result.floor_info.indexOf(")")) + } + height = height.replace("共", "").replace("层", "") + if (height) { + result.height = height.toInteger() + } + } + + + if (result.area && result.total_price) { + result.avg_price = (result.total_price / result.area).toInteger() + } + + def fitmitStr = JQueryUtils.selectText(".maininfo-model-item-2 .maininfo-model-weak", doc) + result.fitment = DataCrawlUtils.parseFitment(fitmitStr) + + + def dateStr = JQueryUtils.selectText(["td:contains(发布时间) .houseInfo-main-item-name"], doc) + + + } + } + + def getDocument(url, driver){ + def stop = false + def error = 0 + def doc = null + while(!stop){ + try{ + driver.get(url) + doc = Jsoup.parse(driver.getPageSource()); + Thread.sleep(5000) + def els = JQueryUtils.select(".list-main .list-left .list", doc) + if(!els){ + els = JQueryUtils.select(".props-main", doc) + } + if(els && els.size()>0){ + stop = true; + }else{ + error++ + driver.manage().deleteAllCookies() + println "[${error}] error: "+url + Thread.sleep(5000) + } + }catch(Exception e){ + error++ + } + if(error>=20){ + stop = true; + } + } + return doc + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeToDB.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeToDB.groovy new file mode 100644 index 000000000..85060a709 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeToDB.groovy @@ -0,0 +1,631 @@ +package com.fangjia.script.gj.crawl + +import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.rabbitmq.client.impl.AMQImpl +import net.sf.json.JSONObject +import org.bson.types.ObjectId + +class AnjukeToDB { + + DBCollection detail = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "app_anjuke_detail") + + static main(args) { + def task = new AnjukeToDB(); + task.run(); +// task.addNoMatch() +// task.updateEstateType() +// task.gaodeRangeData() + } + + def updateEstateType(){ + def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\安居客重庆小区-曹玉彦.xlsx",0,1) + for(def row : datas){ + def anjukeId = row['A'] + def obj = findAnjukeDistrictDetail(anjukeId) + if(obj) { + def type = getEstatType(obj.extend.type) + def commId = row['L'] + if(!commId){ + commId = row['G'] + } + if(type){ + if(type=="别墅"){ + def udata = new BasicDBObject() + udata.put("estate_type2", type) + udata.put("m_date", new Date()) + println commId + DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(commId.trim())), new BasicDBObject('$set', udata)) + } + }else{ +// def udata = new BasicDBObject() +// udata.put("status", 1) +// udata.put("name_cat",11) +// udata.put("m_date", new Date()) +// DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(commId)), new BasicDBObject('$set', udata)) + + } + } + } + } + + def gaodeRangeData(){ + def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\徐鹤高德地图.xlsx",0,1) + def index = 0 + for(def d : datas){ + index++ + println index + def personName = d['AB'] + if(personName){ + def commid = d['J'] + if(commid){ + def obj = CommonDatautils.findDistrict("重庆",d['C'],personName,"district") + if(obj){ + updateDist(d, obj._id.toString()) + } + }else{ + saveGaode(d) + } + }else{ + def commid = d['J'] + updateDist(d, commid) + } + + } + } + + def saveGaode(d){ + def oldName = d['E'] + def distName = nameFileter(d['C'], oldName) + def udata = new BasicDBObject() + def alias = [] + + def aliasStr = d['F'] + if(aliasStr){ + def arrs = aliasStr.split("/") + for(def s : arrs){ + alias << s + } + } + if(alias.size()>0){ + udata.put("alias", alias) + } + + def address = AddressUtil.formatAddress("重庆", d['G']) + if(address){ + udata.put("address", address) + } + if(d['N']){ + udata.put("virescence_percent", d['N']) + } + + if(d['O']){ + udata.put("estate_charge", d['O']) + } + if(d['Q']){ + udata.put("cubage_percent", d['Q']) + } + if(d['R']){ + udata.put("developer", d['R']) + } + + if( d['S']){ + udata.put("build_area", d['S']) + } + + if(d['T']){ + def t = d['T'].substring(0, 3) + udata.put("build_date", t) + } + + if(d['V'] && d['V'].isNumber()){ + udata.put("car_site_count", d['V'].toInteger()) + } + + if(d['Y']){ + def build_type = getBuildType2(d['Y']) + if(build_type){ + udata.put("build_type", build_type) + } + } + + if(d['Z']){ + udata.put("property_life", d['Z']) + } + + def lat = null + def lng = null + if(d['H']){ + lat = d['H'] + lng = d['I'] + // 高德转百度 + def baidu = BaiDuApiUtils.geoconv(lng.toDouble(), lat.toDouble(), 3) + def lng2 = baidu.x + def lat2 = baidu.y + + Long dLat2 = lat2 * 10000000000L + Long dLng2 = lng2 * 10000000000L + udata.put("lat2", dLat2) + udata.put("lng2", dLng2) + Long dlat = lat.toDouble() * 10000000000L + Long dlng = lng.toDouble() * 10000000000L + udata.put("lat", dlat) + udata.put("lng", dlng) + } + def type = "普通住宅" + if(distName.contains("别墅")) { + type = "别墅" + }else if(distName.contains("商住")){ + type = "商住" + } + + if(udata){ + udata.put("name_cat",0) + udata.put("status",0) + udata.put("cat","district") + udata.put("c_date", new Date()) + udata.put("m_date", new Date()) + udata.put("city", "重庆") + udata.put("region",d['C']) + udata.put("name", distName) + udata.put("estate_type2",type) + + def markMap = [:] + markMap.put("data_source", "机器入库") + udata.put("mark_map",markMap) + DbUtils.seaweed.save(udata); + + def stop = false + while (!stop){ + if(udata._id.toString()){ + addPrice(udata) + stop = true; + } + } + } + } + + def updateDist(d, commid){ + + if(commid){ + def dist = CommonDatautils.queryDistrictById(commid) + if(dist){ + def oldName = d['E'] + def distName = nameFileter(d['C'], oldName) + def udata = new BasicDBObject() + if(!distName.contains("号")){ + def alias = dist.alias + if(!alias){ + alias = [] + } + if(!alias.contains(distName) && dist.name!=distName){ + alias << distName + } + def aliasStr = d['F'] + if(aliasStr){ + def arrs = aliasStr.split("/") + for(def s : arrs){ + alias << s + } + } + if(alias.size()>0 && alias!=dist.alias){ + udata.put("alias", alias) + } + } + + if(dist.address && !dist.address.contains("号")){ + def address = AddressUtil.formatAddress("重庆", d['G']) + if(address){ + udata.put("address", address) + } + } + + if(!dist.virescence_percent && d['N']){ + udata.put("virescence_percent", d['N']) + } + + if(!dist.estate_charge && d['O']){ + udata.put("estate_charge", d['O']) + } + if(!dist.cubage_percent && d['Q']){ + udata.put("cubage_percent", d['Q']) + } + if(!dist.developer && d['R']){ + udata.put("developer", d['R']) + } + + if(!dist.build_area && d['S']){ + udata.put("build_area", d['S']) + } + + if(!dist.build_date && d['T']){ + if(d['T'].size()>=4){ + def t = d['T'].substring(0, 4) + udata.put("build_date", t) + } + + } + + if(!dist.car_site_count && d['V'] && d['V'].isNumber()){ + udata.put("car_site_count", d['V'].toInteger()) + } + + if(!dist.build_type && d['Y']){ + def build_type = getBuildType2(d['Y']) + if(build_type){ + udata.put("build_type", build_type) + } + } + + if(!dist.property_life && d['Z']){ + udata.put("property_life", d['Z']) + } + + def lat = null + def lng = null + if(d['H']){ + lat = d['H'] + lng = d['I'] + + def slat = dist.lat/10000000000.0 + def slng = dist.lng/10000000000.0 + + def distance = GeoUtils.calculateDistance(lng.toDouble(), lat.toDouble(), slng, slat); + if(distance>1000){ + // 高德转百度 + def baidu = BaiDuApiUtils.geoconv(lng.toDouble(), lat.toDouble(), 3) + def lng2 = baidu.x + def lat2 = baidu.y + + Long dLat2 = lat2 * 10000000000L + Long dLng2 = lng2 * 10000000000L + udata.put("lat2", dLat2) + udata.put("lng2", dLng2) + Long dlat = lat.toDouble() * 10000000000L + Long dlng = lng.toDouble() * 10000000000L + udata.put("lat", dlat) + udata.put("lng", dlng) + } + + } + + if(udata){ + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) + println "[修改] ${dist.name}"+ udata + } + } + } + } + + def addNoMatch(){ + def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\安居客未匹配.xlsx",0,1) + for(def d : datas){ + add(d) + } + } + def run(){ + def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\非人工处理.xlsx",0,1) + for(def data : datas){ +// if(data['K']=="镕杰"){ + def flag = data['M'] + if(flag && flag=="新增"){ +// add(data) + }else{ + update(data) + } +// } + } + } + + + def add(def row){ + def anjukeId = row['A'] + def obj = findAnjukeDistrictDetail(anjukeId) + if(obj){ + def type = getEstatType(obj.extend.type) + if(type){ + def city = obj.base.cityName + def region = obj.base.areaName + def lat2 = null + def lng2 = null + def lat = null + def lng = null + if(obj.base.lat){ + lat2 = obj.base.lat + lng2 = obj.base.lng + + JSONObject convertObj = GaoDeApiUtils.convert(lng2+","+lat2, "baidu") + if(convertObj.getInt("status") == 1) { + String lo = convertObj.get("locations") + lng = Double.parseDouble(lo.split(",")[0]) + lat = Double.parseDouble(lo.split(",")[1]) + } + } + + def distName = nameFileter(obj.base.areaName, obj.base.name) + + def buildTypeStr = getBuildType(obj.base.buildTypeStr) + + def address = AddressUtil.formatAddress(obj.base.city, obj.base.address) + + def dist = CommonDatautils.findDistrict(city, region, distName, "district") + if(!dist){ + def data = new BasicDBObject(); + data.put("name_cat",0) + data.put("status",0) + data.put("cat","district") + data.put("c_date", new Date()) + data.put("m_date", new Date()) + data.put("city", city) + data.put("region",region) + data.put("name", distName) + data.put("estate_type2",type) + if(address){ + data.put("address", obj.base.address) + } + + if(lat){ + Long alat = lat*10000000000L + Long alng = lng*10000000000L + data.put("lat", alat) + data.put("lng", alng) + Long alat2 = lat2.toDouble()*10000000000L + Long alng2 = lng2.toDouble()*10000000000L + data.put("lat2",alat2) + data.put("lng2",alng2) + } + data.put("finish_building_date",obj.extend.completionTime) + if(obj.extend.totalHouseHoldNum && obj.extend.totalHouseHoldNum.isNumber()){ + data.put("household_count",obj.extend.totalHouseHoldNum.toInteger()) + } + data.put("build_area", obj.extend.totalArea) + + data.put("virescence_percent", obj.extend.landscapingRatio) + data.put("build_type", buildTypeStr) + data.put("business_quarter",obj.base.tradingAreaName) + + + def markMap = [:] + markMap.put("data_source", "机器入库") + data.put("mark_map",markMap) + DbUtils.seaweed.save(data); + + def stop = false + while (!stop){ + if(data._id.toString()){ + addPrice(data) + stop = true; + } + } + + } + }else{ + println "【物业类型问题】 ${obj._id.toString()} "+obj.extend.type + } + }else{ + println anjukeId+" 没找到" + } + + } + def update(def row){ + def commId = row['L'] + if(!commId){ + commId = row['G'] + } + + def dist = CommonDatautils.queryDistrictById(commId) + def anjukeId = row['A'] + def obj = findAnjukeDistrictDetail(anjukeId) + if(obj && dist){ + def udata = new BasicDBObject() + + def address = AddressUtil.formatAddress(obj.base.cityName, obj.base.address) + if(address){ + if(!dist.address){ + if(obj.base.address){ + udata.put("address",obj.base.address) + } + }else{ + if(dist.address.contains("号")){ + if(!dist.address.contains(obj.base.address) && !obj.base.address.contains(dist.address)){ + def addr_alias = dist.addr_alias + if(!addr_alias){ + addr_alias = [] + } + if(!addr_alias.contains(obj.base.address) && obj.base.address.contains("号")){ + addr_alias << obj.base.address + udata.put("addr_alias", addr_alias) + } + } + }else{ + udata.put("address",obj.base.address) + } + } + } + + def distName = nameFileter(obj.base.areaName, obj.base.name) + if(!distName.contains("号")){ + def alias = dist.alias + if(!alias){ + alias = [] + } + if(alias.contains(distName)){ + alias << distName + } + if(alias.size()>0 && alias!=dist.alias){ + udata.put("alias", alias) + } + } + + if(!dist.build_type){ + def buildTypeStr = getBuildType(obj.base.buildTypeStr) + if(buildTypeStr){ + udata.put("build_type",buildTypeStr) + } + + } + if(!dist.household_count){ + if(obj.extend.totalHouseHoldNum && obj.extend.totalHouseHoldNum.isNumber() && obj.extend.totalHouseHoldNum.toInteger()>0){ + udata.put("household_count",obj.extend.totalHouseHoldNum.toInteger()) + } + } + if(!dist.finish_building_date){ + if(obj.extend.completionTime){ + udata.put("finish_building_date",obj.extend.completionTime) + } + } + + if(!dist.virescence_percent){ + if(obj.extend.landscapingRatio){ + udata.put("virescence_percent", obj.extend.landscapingRatio) + } + } + if(!dist.business_quarter){ + if(obj.base.tradingAreaName){ + udata.put("business_quarter",obj.base.tradingAreaName) + } + } + def lat2 = null + def lng2 = null + if(dist.lat2){ + lat2 = dist.lat2/10000000000.0 + lng2 = dist.lng2/10000000000.0 + } + if(dist.lat && !lat2){ + // 高德转百度 + def baidu = BaiDuApiUtils.geoconv(dist.lng/10000000000.0, dist.lat/10000000000.0, 3) + lng2 = baidu.x + lat2 = baidu.y + + Long dLat2 = lat2 * 10000000000L + Long dLng2 = lng2 * 10000000000L + udata.put("lat2", dLat2) + udata.put("lng2", dLng2) + } + + if(obj.base.lat) { + def alat2 = obj.base.lat.toDouble() + def alng2 = obj.base.lng.toDouble() + def distance = GeoUtils.calculateDistance(lng2, lat2, alng2, alat2); + if(distance>1000){ + println "........${obj.base.name}..............${dist.name}..........距离相差${distance}......................" + + def lng = null + def lat = null + JSONObject convertObj = GaoDeApiUtils.convert(obj.base.lng+","+obj.base.lat, "baidu") + if(convertObj.getInt("status") == 1) { + String lo = convertObj.get("locations") + lng = Double.parseDouble(lo.split(",")[0]) + lat = Double.parseDouble(lo.split(",")[1]) + } + + Long blat2 = alat2 *10000000000L + Long blng2 = alng2 *10000000000L + Long gLat2 = lat *10000000000L + Long gLng2 = lng *10000000000L + + udata.put("lat2", blat2) + udata.put("lng2", blng2) + udata.put("lat", gLat2) + udata.put("lng", gLng2) + } + } + + if(udata){ + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) + println "[修改] ${dist.name}"+ udata + } + + } + } + + def findAnjukeDistrictDetail(anjukeId){ + def q = new BasicDBObject() + try{ + q.put("_id",new ObjectId(anjukeId)) + return detail.findOne(q) + }catch(Exception e){ + print "[查询出错]anjukeId:"+anjukeId + } + return null + } + + def nameFileter(def region, def name){ + name = name.replace("·","").replace(".","").replace("|","") + name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期") + name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区") + name = name.replace("(${region})","").replace("(","").replace(")","") + return name; + } + + def getEstatType(type){ + if(type){ + if(type=="公寓住宅"){ + type = "普通住宅" + }else if(type.contains("商住")){ + type = "商住" + }else if(type.contains("别墅")){ + type = "别墅" + }else{ + type = null + } + } + return type + } + + def getBuildType(def buildType){ + def types = ["多层","小高层","高层","超高层","花园洋房","老洋房","四合院","独栋","双拼","联排","叠加","老式别墅洋房"] + def list = [] + if(buildType){ + def arrs = buildType.split("\\|") + for(def s : arrs){ + if(types.contains(s)){ + list << s + } + } + if(list){ + return list.join(",") + } + } + return null + } + + def getBuildType2(def buildType){ + def types = ["多层","小高层","高层","超高层","花园洋房","老洋房","四合院","独栋","双拼","联排","叠加","老式别墅洋房"] + def list = [] + if(buildType){ + def arrs = buildType.split(",| ") + for(def s : arrs){ + if(types.contains(s)){ + list << s + } + } + if(list){ + return list.join(",") + } + } + return null + } + + def addPrice(dist){ + def startDate = DateUtils.parseFromTightYYYYMMDD("2022-04-01") + def endDate = DateUtils.parseFromTightYYYYMMDD("2022-05-25") + def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate) + + for(def weekend : weekends){ + CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0) + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/DataCrawlUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/DataCrawlUtils.groovy new file mode 100644 index 000000000..465ae5f04 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/DataCrawlUtils.groovy @@ -0,0 +1,250 @@ +package com.fangjia.script.gj.crawl + +import com.fangjia.script.util.DateUtils +import org.apache.commons.lang.StringUtils + +import java.util.regex.Matcher +import java.util.regex.Pattern + +class DataCrawlUtils { + static final Map DEAL_DATE_RESOLVERS = [:] + static final Map ROOM_RESOLVERS = [:] + static final Map HALL_RESOLVERS = [:] + static final Map TOILET_RESOLVERS = [:] + static final Map TOTAL_PRICE_RESOLVERS = [:] + static final Map UNIT_PRICE_RESOLVERS = [:] + static final Map AREA_RESOLVERS = [:] + static final Map HEIGHT_RESOLVERS = [:] + static final Map FLOOR_RESOLVERS = [:] + static final List DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"] + private static final HashMap> FITMENTS_TYPES_MAPS = new HashMap>() + + static { + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}-\\d{2}"), { String str -> + return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd") + }) + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}-\\d{2}-\\d{2}"), { String str -> + str = "20" + str + return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd") + }) + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}/\\d{2}/\\d{2}"), { String str -> + str = "20" + str + return DateUtils.parseFromTightByPattern(str, "yyyy/MM/dd") + }) + + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}\\.\\d{2}\\.\\d{2}"), { String str -> + return DateUtils.parseFromTightByPattern(str, "yyyy.MM.dd") + }) + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}"), { String str -> + return DateUtils.parseFromTightByPattern(str, "yyyy-MM") + }) + + ROOM_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两](室|房)"), { String str -> + str = str.replaceAll("室|房", '').trim() + return str.toInteger() + }) + + HALL_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]厅"), { String str -> + str = str.replace("厅", '').trim() + return str.toInteger() + }) + TOILET_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]卫"), { String str -> + str = str.replace("卫", '').trim() + return str.toInteger() + }) + TOTAL_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?万"), { String str -> + str = str.replace("万", '').trim() + return (str.toDouble() * 10000).toInteger() + }) + + UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/㎡"), { String str -> + str = str.replace("元/㎡", '').trim() + return str.toDouble().toInteger() + }) + UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/m?"), { String str -> + str = str.replace("元/m?", '').trim() + return str.toDouble().toInteger() + }) + UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/平"), { String str -> + str = str.replace("元/平", '').trim() + return str.toDouble().toInteger() + }) + UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元"), { String str -> + str = str.replace("元", '').trim() + return str.toDouble().toInteger() + }) + + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平米"), { String str -> + str = str.replace("平米", '').trim() + return str.toDouble() + }) + + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平方米"), { String str -> + str = str.replace("平方米", '').trim() + return str.toDouble() + }) + + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平"), { String str -> + str = str.replace("平", '').trim() + return str.toDouble() + }) + + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?㎡"), { String str -> + str = str.replace("㎡", '').trim() + return str.toDouble() + }) + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?m?"), { String str -> + str = str.replace("m?", '').trim() + return str.toDouble() + }) + HEIGHT_RESOLVERS.put(Pattern.compile("共\\d+层"), { String str -> + str = str.replace("共", '').replace('层', '') + return str.toInteger() + }) + + HEIGHT_RESOLVERS.put(Pattern.compile("总\\d+层"), { String str -> + str = str.replace("总", '').replace('层', '') + return str.toInteger() + }) + + HEIGHT_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str -> + def split = str.split("/") + def n1 = split[0].toInteger() + def n2 = split[1].toInteger() + if (n1 <= n2) { + return n2 + } + return null + }) + + + FLOOR_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str -> + def split = str.split("/") + def n1 = split[0].toInteger() + def n2 = split[1].toInteger() + if (n1 <= n2) { + return n1 + } + return n1 + }) + + FLOOR_RESOLVERS.put(Pattern.compile("第\\d+(层|楼)"), { String str -> + str = str.replace("第", "").replaceAll("层|楼", "") + return str.toInteger() + }) + + FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"]) + FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"]) + FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"]) + FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"]) + + } + + /** + * 解析室数 + * @param content + * @return + */ + static Integer parseRoom(String content) { + return parse(content, ROOM_RESOLVERS) + } + /** + * 解析厅数 + * @param content + * @return + */ + static Integer parseHall(String content) { + return parse(content, HALL_RESOLVERS) + } + /** + * 解析厅数 + * @param content + * @return + */ + static Integer parseToilet(String content) { + return parse(content, TOILET_RESOLVERS) + } + + static Object parse(String content, Map resolvers) { + if (content != null) { + content = content.replaceAll("\\s", "") + } + for (def resolver : resolvers.entrySet()) { + String match = match(content, resolver.getKey()) + if (StringUtils.isNotBlank(match)) { + match = match.replace("一","1").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("两","2") + return resolver.getValue()(match) + } + } + return null + } + + static String match(String str, Pattern pattern) { + if (str == null) { + return null + } + Matcher matcher = pattern.matcher(str) + if (matcher.find()) { + return matcher.group() + } + return null + } + + static String parseDirection(String directionDesc) { + if (directionDesc == null) { + return null + } + directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#") + StringBuffer sb = new StringBuffer() + for (int i = 0; i < directionDesc.length(); i++) { + String c = directionDesc.substring(i, i + 1) + if (DIRECTIONS.contains(c)) { + if (directionDesc.substring(i).startsWith(c + "#_#")) { + sb.append(c + " ") + } else { + sb.append(c) + } + } + } + for (String d : DIRECTIONS) { + if (sb.toString().startsWith(d)) { + return d + } + } + for (String d : DIRECTIONS) { + if (sb.toString().contains(d)) { + return d + } + } + return null + + } + + static String parseFitment(String fitmentDesc) { + if (fitmentDesc == null) { + return null + } + fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "") + if (fitmentDesc.equals("非毛坯")) { + return null + } + Iterator>> it = FITMENTS_TYPES_MAPS + .entrySet().iterator() + Map.Entry> entry + while (it.hasNext()) { + entry = it.next() + if (fitmentDesc.contains(entry.getKey())) { + return entry.getKey() + } + for (String fitmentKey : entry.getValue()) { + if (fitmentDesc.contains(fitmentKey)) { + return entry.getKey() + } + } + } + return null + + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/DealPriceUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/DealPriceUtils.groovy new file mode 100644 index 000000000..02fc9458d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/DealPriceUtils.groovy @@ -0,0 +1,347 @@ +package com.fangjia.script.gj.crawl + +import com.fangjia.script.util.DateUtils +import org.apache.commons.lang.StringUtils + +import java.util.regex.Matcher +import java.util.regex.Pattern + +/** + * 鎴愪氦鏁版嵁瑙f瀽宸ュ叿绫� + */ +class DealPriceUtils { + + static final Map DEAL_DATE_RESOLVERS = [:] + static final Map ROOM_RESOLVERS = [:] + static final Map HALL_RESOLVERS = [:] + static final Map TOILET_RESOLVERS = [:] + static final Map TOTAL_PRICE_RESOLVERS = [:] + static final Map UNIT_PRICE_RESOLVERS = [:] + static final Map AREA_RESOLVERS = [:] + static final Map HEIGHT_RESOLVERS = [:] + static final Map FLOOR_RESOLVERS = [:] + static final List DIRECTIONS = ["涓滃寳", "瑗垮寳", "涓滃崡", "瑗垮崡", "鍗楀寳", "涓滆タ", "涓�", "鍗�", "瑗�", "鍖�"] + private static final HashMap> FITMENTS_TYPES_MAPS = new HashMap>() + + static { + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}-\\d{2}"), { String str -> + return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd") + }) + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}-\\d{2}-\\d{2}"), { String str -> + str = "20" + str + return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd") + }) + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}/\\d{2}/\\d{2}"), { String str -> + str = "20" + str + return DateUtils.parseFromTightByPattern(str, "yyyy/MM/dd") + }) + + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}\\.\\d{2}\\.\\d{2}"), { String str -> + return DateUtils.parseFromTightByPattern(str, "yyyy.MM.dd") + }) + DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}"), { String str -> + return DateUtils.parseFromTightByPattern(str, "yyyy-MM") + }) + + ROOM_RESOLVERS.put(Pattern.compile("[\\d+|涓�|浜寍涓墊鍥泑浜攟涓(瀹鎴�)"), { String str -> + str = str.replaceAll("瀹鎴�", '').trim() + return str.toInteger() + }) + + HALL_RESOLVERS.put(Pattern.compile("[\\d+|涓�|浜寍涓墊鍥泑浜攟涓鍘�"), { String str -> + str = str.replace("鍘�", '').trim() + return str.toInteger() + }) + TOILET_RESOLVERS.put(Pattern.compile("[\\d+|涓�|浜寍涓墊鍥泑浜攟涓鍗�"), { String str -> + str = str.replace("鍗�", '').trim() + return str.toInteger() + }) + TOTAL_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?涓�"), { String str -> + str = str.replace("涓�", '').trim() + return (str.toDouble() * 10000).toInteger() + }) + + UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?鍏�/銕�"), { String str -> + str = str.replace("鍏�/銕�", '').trim() + return str.toDouble().toInteger() + }) + UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?鍏�/m虏"), { String str -> + str = str.replace("鍏�/m虏", '').trim() + return str.toDouble().toInteger() + }) + UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?鍏�/骞�"), { String str -> + str = str.replace("鍏�/骞�", '').trim() + return str.toDouble().toInteger() + }) + UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?鍏�"), { String str -> + str = str.replace("鍏�", '').trim() + return str.toDouble().toInteger() + }) + + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?骞崇背"), { String str -> + str = str.replace("骞崇背", '').trim() + return str.toDouble() + }) + + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?骞虫柟绫�"), { String str -> + str = str.replace("骞虫柟绫�", '').trim() + return str.toDouble() + }) + + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?骞�"), { String str -> + str = str.replace("骞�", '').trim() + return str.toDouble() + }) + + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?銕�"), { String str -> + str = str.replace("銕�", '').trim() + return str.toDouble() + }) + AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?m虏"), { String str -> + str = str.replace("m虏", '').trim() + return str.toDouble() + }) + HEIGHT_RESOLVERS.put(Pattern.compile("鍏盶\d+灞�"), { String str -> + str = str.replace("鍏�", '').replace('灞�', '') + return str.toInteger() + }) + + HEIGHT_RESOLVERS.put(Pattern.compile("鎬籠\d+灞�"), { String str -> + str = str.replace("鎬�", '').replace('灞�', '') + return str.toInteger() + }) + + HEIGHT_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str -> + def split = str.split("/") + def n1 = split[0].toInteger() + def n2 = split[1].toInteger() + if (n1 <= n2) { + return n2 + } + return null + }) + + + FLOOR_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str -> + def split = str.split("/") + def n1 = split[0].toInteger() + def n2 = split[1].toInteger() + if (n1 <= n2) { + return n1 + } + return n1 + }) + + FLOOR_RESOLVERS.put(Pattern.compile("绗琝\d+(灞倈妤�)"), { String str -> + str = str.replace("绗�", "").replaceAll("灞倈妤�", "") + return str.toInteger() + }) + + FITMENTS_TYPES_MAPS.put("姣涘澂", ["姣涘澂", "娓呮按", "鏃犺淇�", "鐙爧姣涘澂", "姣涜儦", "鐧藉澂", "鏃ц", "鍧�", "鍘熸埧", "姣涘潖"]) + FITMENTS_TYPES_MAPS.put("绠�瑁�", ["绠�瑁�", "鏅�氳淇�", "鏅��", "绠�鍗曡淇�", "绠�鏄撹淇�", "鏅", "涓�鑸淇�", "涓�鑸�", "绠�", "绮楄淇�"]) + FITMENTS_TYPES_MAPS.put("绮捐", ["绮捐", "鍏ㄨ淇�", "鍏ㄨ", "涓瓑瑁呬慨", "涓", "涓。瑁呬慨", "绮捐嚧瑁呬慨", "涓。", "涓瓑", "绮�", "涓慨", "浼樿川瑁呬慨", "鏃跺皻瑁呬慨", "鑹", "涓珮鎸�"]) + FITMENTS_TYPES_MAPS.put("璞", ["璞崕瑁呬慨", "楂樻。", "璞崕", "璞", "楂樼骇瑁呬慨", "楂樿", "濂緢"]) + + } + /** + * 瑙f瀽鎴愪氦鏃ユ湡 + * @param content + * @return + */ + static Date parseDealDate(String content) { + return parse(content, DEAL_DATE_RESOLVERS) + } + + /** + * 瑙f瀽瀹ゆ暟 + * @param content + * @return + */ + static Integer parseRoom(String content) { + return parse(content, ROOM_RESOLVERS) + } + /** + * 瑙f瀽鍘呮暟 + * @param content + * @return + */ + static Integer parseHall(String content) { + return parse(content, HALL_RESOLVERS) + } + /** + * 瑙f瀽鍘呮暟 + * @param content + * @return + */ + static Integer parseToilet(String content) { + return parse(content, TOILET_RESOLVERS) + } + + /** + * 瑙f瀽鎬讳环 + * @param content + * @return + */ + static Integer parseTotalPrice(String content) { + return parse(content, TOTAL_PRICE_RESOLVERS) + } + /** + * 瑙f瀽鍗曚环 + * @param content + * @return + */ + static Integer parseUnitPrice(String content) { + return parse(content, UNIT_PRICE_RESOLVERS) + } + /** + * 瑙f瀽闈㈢Н + * @param content + * @return + */ + static Double parseArea(String content) { + return parse(content, AREA_RESOLVERS) + } + /** + * 瑙f瀽妤煎眰 + * @param content + * @return + */ + static Integer parseFloor(String content) { + return parse(content, FLOOR_RESOLVERS) + } + /** + * 瑙f瀽妤奸珮 + * @param content + * @return + */ + static Integer parseHeight(String content) { + return parse(content, HEIGHT_RESOLVERS) + } + + /** + * 鏍煎紡鍖栧皬鍖哄悕 + * @param name + * @return + */ + static String formatName(String name) { + name = name.replace("-鍟嗕綇", "").replace("-鍒", "") + //姹囩瀹跺洯骞冲畨閲岋紙杩侊級2015-2017 + name = name.replaceAll("\\d\\d\\d\\d-\\d\\d\\d\\d", "") + //鍥借锤澶╂偊涓�鏈�2017 + name = name.replaceAll("\\d\\d\\d\\d\$", "") + if (name ==~ /.+(\(|锛�)[\d涓�浜屼笁鍥涗簲鍏竷鍏節鍗佷笢鍗楄タ鍖梶\w]+(鏈焲鍖�)(\)|锛�)$/) { + name = name.replaceAll("\\(|锛坾\\)|锛�", '') + } else if (name ==~ /.+(\(|锛�)鍒(\)|锛�)$/) { + name = name.replaceAll("\\(|锛坾\\)|锛�", '') + } else if (name ==~ /.+(\(|锛�).+(\)|锛�)$/) { + name = name.replaceAll("(\\(|锛�).+(\\)|锛�)", '') + } else if (name.contains("路") || (name.contains("-") && !(name ==~ /.*\d+-\d+.*/))) { + println("---------------------------->" + name) + name = name.replaceAll("路|-", '') + println(name) + } + if (name ==~ /.{2,}璺�.{3,}$/ && !(name ==~ /.{2,}璺�.+鍙�/) && !(name ==~ /.{2,}璺�.+寮�/) && !(name ==~ /.*灏忓尯$/)) { + name = name.replaceAll("^.{2,}璺�", "") + } + return name + } + + + static Object parse(String content, Map resolvers) { + if (content != null) { + content = content.replaceAll("\\s", "") + } + for (def resolver : resolvers.entrySet()) { + String match = match(content, resolver.getKey()) + if (StringUtils.isNotBlank(match)) { + match = match.replace("涓�","1").replace("浜�","2").replace("涓�","3").replace("鍥�","4").replace("浜�","5").replace("涓�","2") + return resolver.getValue()(match) + } + } + return null + } + + /** + * 鍖归厤绗﹀悎鏉′欢鐨勫瓧绗︿覆 + * + * @param str + * @param pattern + * @return + */ + static String match(String str, Pattern pattern) { + if (str == null) { + return null + } + Matcher matcher = pattern.matcher(str) + if (matcher.find()) { + return matcher.group() + } + return null + } + + static String parseFitment(String fitmentDesc) { + if (fitmentDesc == null) { + return null + } + fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "") + if (fitmentDesc.equals("闈炴瘺鍧�")) { + return null + } + Iterator>> it = FITMENTS_TYPES_MAPS + .entrySet().iterator() + Map.Entry> entry + while (it.hasNext()) { + entry = it.next() + if (fitmentDesc.contains(entry.getKey())) { + return entry.getKey() + } + for (String fitmentKey : entry.getValue()) { + if (fitmentDesc.contains(fitmentKey)) { + return entry.getKey() + } + } + } + return null + + } + + static String parseDirection(String directionDesc) { + if (directionDesc == null) { + return null + } + directionDesc = directionDesc.replaceAll("(\\(|锛坾銆妡\"|鈥渱銆恷銆�).+?(\\)|銆媩锛墊\"|鈥潀銆憒銆�)", "#_#") + StringBuffer sb = new StringBuffer() + for (int i = 0; i < directionDesc.length(); i++) { + String c = directionDesc.substring(i, i + 1) + if (DIRECTIONS.contains(c)) { + if (directionDesc.substring(i).startsWith(c + "#_#")) { + sb.append(c + " ") + } else { + sb.append(c) + } + } + } + for (String d : DIRECTIONS) { + if (sb.toString().startsWith(d)) { + return d + } + } + for (String d : DIRECTIONS) { + if (sb.toString().contains(d)) { + return d + } + } + return null + + } + + + static String parsePrice(String priceStr) { + def pattern =Pattern.compile("\\d+(\\.\\d+)?涓�") + String match = match(priceStr, pattern) + } + + static void main(String[] args) { + println parsePrice("椤剁娆e洯瑗胯嫅2瀹�1鍘�1鍗�492.3涓�") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy new file mode 100644 index 000000000..845ab47d0 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy @@ -0,0 +1,167 @@ +package com.fangjia.script.gj.crawl + +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil +import com.fangjia.script.thirdPriceCrawl.Utils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.jsoup.Jsoup +import org.openqa.selenium.By +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.ChromeDriver + +class ZhugeFangDistCrawl { + + public static final SHORT_DATE = Utils.getLastMonth(); + def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); + + static main(args) { + def task = new ZhugeFangDistCrawl(); + task.run(); + } + def run(){ + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + def url = "https://fz.esfxiaoqu.zhuge.com/fuqing/" + WebDriver driver = new ChromeDriver() + driver.manage().deleteAllCookies() + getRegions(driver, url) + } + + def getRegions(driver, url) { + for(def i=1;i<=11;i++){ + def newUrl = url+"page${i}/" + println newUrl + def doc = getDocument(newUrl, driver) + def total = parseList(doc) + +// def next = JQueryUtils.select("a:contains(下一页)", doc) +// if(next){ +// driver.findElement(By.xpath("//a[contains(text(), '下一页')]")).click() +// } + + } + + } + + def parseList(doc){ + def city= "福州" + def els = JQueryUtils.select("#listTableBox li", doc) + if (els) { + for(def e : els){ + def name = e.select(".house-name").text() + def str = e.select(".house-adr").text() + def address = null + if(str){ + if(str.contains("]")){ + address = str.substring(str.indexOf("]")+1).trim() + }else{ + address = str + } + } + def regionStr = null + if(str && str.contains("]") && str.contains("[")){ + regionStr = str.substring(str.indexOf("[")+1, str.indexOf("]")) + } + + def region = null + def block = null + if(regionStr){ + def arrs = regionStr.split("-") + if(arrs && arrs.size()>0){ + if(arrs.size()==2){ + block = arrs[1].trim() + } + if(arrs.size()>=1){ + region = arrs[0].trim() + } + + } + } + def links = e.select(".house-name a") + def href = null + if(links){ + href = links[0].attr("href") + } + def priceStr = e.select(".price-number").text() + def price = null + if(priceStr && !priceStr.contains("暂无")){ + if(priceStr.contains("元")){ + priceStr = priceStr.substring(0, priceStr.indexOf("元")).trim() + } + if(priceStr && priceStr.isNumber()){ + price = priceStr.toInteger() + } + } + def obj = findByParams(city, region, name) + if(obj){ + if(price){ + DBObject udata2 = new BasicDBObject() + udata2.put("avg_price", price) + udata2.put("m_date", new Date()) + println "[upeate] ${region} "+name + thirdPartyColl.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata2)); + } + }else{ + def udata = new BasicDBObject() + udata.put("city", city) + udata.put("region", region) + udata.put("block", block) + udata.put("url", href) + udata.put("address", address) + udata.put("name", name) + udata.put("avg_price", price) + udata.put("s_date", SHORT_DATE) + udata.put("c_date", new Date()) + udata.put("m_date", new Date()) + udata.put("source", "诸葛找房") + thirdPartyColl.save(udata) + println "[save] ${region} "+name+" "+priceStr + } + } + } + return els.size() + } + + + def findByParams(city, region, name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put("source", "诸葛找房") + q.put("s_date", SHORT_DATE) + return thirdPartyColl.findOne(q); + } + + + def getDocument(url, driver){ + def stop = false + def error = 0 + def doc = null + while(!stop){ + try{ + driver.get(url) + doc = Jsoup.parse(driver.getPageSource()); + Thread.sleep(10000) + def els = JQueryUtils.select("#listTableBox li", doc) + if(els && els.size()>0){ + stop = true; + }else{ + error++ + driver.manage().deleteAllCookies() + println "[${error}] error: "+url + Thread.sleep(5000) + } + }catch(Exception e){ + error++ + } + if(error>=20){ + stop = true; + } + } + return doc + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy index 71657eea3..cbee24228 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy @@ -172,7 +172,8 @@ class NameProcess { if((name.endsWith("村") || name.endsWith("街"))&& name.size()<=3){ return null } - if(name.endsWith("电梯")||name.endsWith("大道")||name.endsWith("厂") || name.endsWith("组") || (name.endsWith("阁") && name.length()<=3) || name.endsWith("条")){ + //|| name.endsWith("组") + if(name.endsWith("电梯")||name.endsWith("大道")||name.endsWith("厂") || name.endsWith("条")){ return null } if(name.contains("、")||name.contains("|")||name.contains(".")){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy index bd387ed77..3a572c7ad 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -71,7 +71,8 @@ class DistrictIndexBatch extends BaseBatch{ def run(){ def cityCodeDatas = CcbCommonUtils.findCityCodeValue() - def citys = CommonDatautils.findAllCity(); + def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"] + //def citys = CommonDatautils.findAllCity(); for(def city : citys){ def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2","household_count"]) def code = cityCodeDatas.get(city) @@ -86,7 +87,7 @@ class DistrictIndexBatch extends BaseBatch{ city_avg_total_price = 531.2615913 } if(city_avg_total_price){ - // calcute(city, dists, code, city_avg_total_price) + calcute(city, dists, code, city_avg_total_price) calcuteIndex(city, dists, code, city_avg_total_price) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictMapMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictMapMatch.groovy new file mode 100644 index 000000000..47c9fe748 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictMapMatch.groovy @@ -0,0 +1,123 @@ +package com.fangjia.script.gj.location + +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ExcelUtils + +class DistrictMapMatch { + + static main(args) { + def task = new DistrictMapMatch() + task.mapMatch() + } + + + def mapMatch(){ + def index = 0 + ExcelUtils.modify("D:\\数据\\重庆农商行\\小区\\待审核小区(地图匹配).xlsx",{ d-> + if(!d['H']){ + index++ + def rets = OpenApiUtils.getGaoDeSearchLocation(d['A'],d['B'],d['C']) + def map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets); +// if (!map || !map.lat) { +// rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']); +// map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets); +// } + if(map){ + d['F'] = map.id + if(map.adname){ + d['G'] = map.adname + }else if(map.region){ + d['G'] = map.region + } + d['H'] = map.name + d['I'] = map.address + if(map.type){ + d['J'] = map.type + }else if(map.estateType){ + d['J'] = map.estateType + } + d['K'] = map.lng + d['L'] = map.lat + d['M'] = map.score + d['N'] = map.source + } + } + },0) + } + + + def findStandardObject2(def city, def region, def name, def address, def rets) { + if (rets) { + def bestObj = [:] + if (rets.size() >= 1) { + for (def mapObj : rets) { + def score = 0 + if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){ + score = 0.2 + } + if(mapObj.name && name){ + def newName = mapObj.name; + def names = [] + if(mapObj.name && mapObj.name.indexOf("(")>-1){ + def name1 = mapObj.name.substring(0, mapObj.name.indexOf("(")) + names << name1 + def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")")) + names << name2 + }else{ + names << newName + } + def maxNameScore = 0 + for(def n : names){ + double score1 = CosineSimilarAlgorithm.getSimilarity(n, name); + if(score1 && score1>0){ + def nameScore = (score1 * 4)/10 + if(nameScore > maxNameScore){ + maxNameScore = nameScore + } + } + } + + if(maxNameScore){ + try{ + maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore) + }catch(Exception e){ + maxNameScore = 0 + } + + } + score = maxNameScore + score; + } + + if(mapObj.address && address){ + double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address); + if(addressScore && addressScore>0){ + try{ + addressScore = DataHandleUtils.getDoubleRound(2, addressScore) + }catch(Exception e){ + addressScore = 0 + } + score = addressScore*4/10 + score + } + } + + if(!bestObj){ + mapObj.score = score + bestObj = mapObj + }else{ + if(score > bestObj.score){ + mapObj.score = score + bestObj = mapObj + } + } + + } + } + return bestObj + } + return null + + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meituan/MatchTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meituan/MatchTest.groovy index 6b4872d19..f42431df6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meituan/MatchTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meituan/MatchTest.groovy @@ -21,10 +21,58 @@ class MatchTest { def task = new MatchTest() // task.run() // task.addNum() - task.dongguang() +// task.dongguang() // task.meituanDel() + + task.addLevel() } + + + def addLevel() { + def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"]// + def weekend = DateUtils.getFirstDayOfWeek(new Date()); + def cityLevelMap = [:] + for(def city : citys){ + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region","name"]) + def priceMap = CommonDatautils.queryDistrictPrice(city, weekend) + def datas = addPriceData(dists, priceMap); + datas = calcuteLevel(datas); + cityLevelMap.put(city, ["price":priceMap, "level":datas]) + println city +" 楼盘评级同步完成" + } + + + def path = "D:\\数据\\美团\\10城小区明细-结果.xlsx"//美团金融小区测试list-所有数据-评级.xlsx + ExcelUtils.multiThreadingModify(path, { row -> + + def name = row['D'] + def address = row['E'] + def city = RegionData.formatCity(row['A']) + + if(name && !row['O']){ + if(city){ + def map = cityLevelMap.get(city) + def rMap = null + if(map){ + rMap = getArroundDistrictPriceByAddress(city, address, map.level) + } + + if(rMap){ + if(rMap.rank){ + row['Q'] = rMap.rank + } +// if(rMap.level){ +// row['Q'] = rMap.level +// } + } + } + } + + }) + } + + def meituanDel(){ def path = "D:\\数据\\美团\\美团需入库小区合并后.xlsx" def index = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index a6b739898..60ae682b9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -122,7 +122,7 @@ public class BeiKeCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl }else{ @@ -153,7 +153,7 @@ public class BeiKeCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd") if(dealDate && dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e) @@ -193,7 +193,7 @@ public class BeiKeCrawlByChrome { detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger() } if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) { - detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")) + detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")).toInteger() } } @@ -215,7 +215,8 @@ public class BeiKeCrawlByChrome { detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd") } detail.source = "贝壳找房" - def dbo = findTrade(detail) + def detailUrl = e.select(".title a").attr("href") + def dbo = findTrade(detailUrl) if (!dbo) { def houseInfo = e.select(".houseInfo").text() def houseInfos = houseInfo.split("\\|") @@ -228,7 +229,7 @@ public class BeiKeCrawlByChrome { detail.height = floorStr.toInteger() } } - detail.url = e.select(".title a").attr("href") + detail.url = detailUrl detail.create_date = new Date() detail.m_date = new Date() println detail @@ -313,8 +314,8 @@ public class BeiKeCrawlByChrome { def roomStr = JQueryUtils.selectText("li:contains(房屋户型)", doc).replace("房屋户型", "") if (roomStr) { detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger() - detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")) - detail.toilet = roomStr.substring(roomStr.indexOf("卫") - 1, roomStr.indexOf("卫")) + detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")).toInteger() + detail.toilet = roomStr.substring(roomStr.indexOf("卫") - 1, roomStr.indexOf("卫")).toInteger() } detail.direction = JQueryUtils.selectText("li:contains(房屋朝向)", doc).replace("房屋朝向", "") detail.fitment = JQueryUtils.selectText("li:contains(装修情况)", doc).replace("装修情况", "") @@ -336,13 +337,14 @@ public class BeiKeCrawlByChrome { } - def findTrade(def detail){ + def findTrade(def detailUrl){ def q = new BasicDBObject() q.put("city", detail.city) - q.put("region", detail.region) - q.put("district_name", detail.district_name) - q.put("area", detail.area) - q.put("total_price", detail.total_price) +// q.put("region", detail.region) +// q.put("district_name", detail.district_name) +// q.put("area", detail.area) +// q.put("total_price", detail.total_price) + q.put("url", detailUrl) q.put("trade_date", detail.trade_date) q.put("source", detail.source) def dbo = DEAL_PRICE_COLL.findOne(q) @@ -358,13 +360,13 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = e.attr("href") - if(e.text().contains("重庆")){ +// if(e.text().contains("福州")){ def cityMap = [:] cityMap.city = e.text() cityMap.url = "https:"+href list << cityMap //cityMap.put(e.text(), "https:"+href) - } +// } } } return list diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy index 3ea9f1d4a..e11e01e8f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy @@ -29,7 +29,7 @@ class JdPaimaiCrawl { task.run(); } def run(){ - def existsIds = findAllIds() + def existsIds = []//findAllIds() println "已存在:"+existsIds.size() System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); System.setProperty("http.nonProxyHosts", "localhost"); @@ -43,78 +43,80 @@ class JdPaimaiCrawl { def getList(driver){ def list = [] - for(def i=1;i<=32;i++){ - if(i==8){ - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=560&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=573&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=579&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=584&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=589&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=593&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=598&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=604&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=609&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=613&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=617&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=621&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=632&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=6858&_=1564047626268" - }else if(i==5){ - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=142&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=148&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=164&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=199&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=224&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=239&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=248&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=258&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=264&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=274&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=275&_=1564047626268" - }else if(i==17){ - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1432&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1441&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1458&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1466&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1475&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1477&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1479&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=3154&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1381&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1387&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1396&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1405&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1413&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1421&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2922&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2980&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2983&_=1564047626268" - }else if(i==19){ - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1601&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1611&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1617&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1627&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1634&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1709&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1643&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1650&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1655&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1666&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1672&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1677&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1684&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1690&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1698&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1704&_=1564047626268" - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1705&_=1564047626268" - }else{ - list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&_=1564047626268" - } - } + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId=4&_=1564047626268" +// for(def i=1;i<=32;i++){ +// if(i==8){ +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=560&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=573&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=579&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=584&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=589&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=593&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=598&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=604&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=609&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=613&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=617&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=621&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=632&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=6858&_=1564047626268" +// }else if(i==5){ +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=142&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=148&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=164&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=199&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=224&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=239&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=248&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=258&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=264&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=274&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=275&_=1564047626268" +// }else if(i==17){ +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1432&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1441&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1458&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1466&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1475&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1477&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1479&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=3154&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1381&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1387&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1396&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1405&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1413&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1421&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2922&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2980&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2983&_=1564047626268" +// }else if(i==19){ +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1601&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1611&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1617&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1627&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1634&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1709&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1643&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1650&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1655&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1666&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1672&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1677&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1684&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1690&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1698&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1704&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1705&_=1564047626268" +// }else{ +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&_=1564047626268" +// } + +// } def links = [] for(def url : list){ driver.get(url) @@ -430,10 +432,10 @@ class JdPaimaiCrawl { def s1 = JQueryUtils.selectText(["li:contains(竞买公告) .floor-content"], doc) if(s1) { - def start = s1.indexOf("法院将于") + def start = s1.indexOf("将于") def end = s1.indexOf("止") if(start>-1 && end>-1 && end>start){ - s1 = s1.substring(s1.indexOf("法院将于"), s1.indexOf("止")) + s1 = s1.substring(s1.indexOf("将于"), s1.indexOf("止")) } def dateRangs = DataUtil.getDateRange(s1) if(dateRangs && dateRangs.size()==2) { @@ -476,6 +478,11 @@ class JdPaimaiCrawl { } } + + if(!result.startDate){ + def sTime = JQueryUtils.selectText(".startTime", doc).replace("开拍时间:","") + result.startDate = DateUtils.parseFromTightByPattern(sTime, "yyyy年MM月dd日") + } if(result.finishDate){ result.finishDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.finishDate, "2000-01-01") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 6f99480ab..63ecf3c2a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -38,6 +38,26 @@ public class LianjiaCrawlByChrome { // task.run1() task.addDealRegion(); task.standardRegion() +// task.updadteDealPriceField() + } + + def updadteDealPriceField(){ + def citys = CommonDatautils.findAllCity() + def index = 0 + for(def city : citys){ + index ++ + def q = new BasicDBObject() + q.put("city", city) + q.put("hall", "0") + def count = DEAL_PRICE_COLL.count(q) + println "[${index}] "+city +" "+count + if(count && count>0){ + def udata = new BasicDBObject() + udata.put("hall", 0) + DEAL_PRICE_COLL.update(q, new BasicDBObject('$set', udata), false, true); + println "[修改]"+city +" "+ count + } + } } def run1(){ @@ -187,7 +207,7 @@ public class LianjiaCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl }else{ @@ -218,7 +238,7 @@ public class LianjiaCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e) @@ -258,7 +278,7 @@ public class LianjiaCrawlByChrome { detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger() } if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) { - detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")) + detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")).toInteger() } } @@ -378,8 +398,8 @@ public class LianjiaCrawlByChrome { def roomStr = JQueryUtils.selectText("li:contains(房屋户型)", doc).replace("房屋户型", "") if (roomStr) { detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger() - detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")) - detail.toilet = roomStr.substring(roomStr.indexOf("卫") - 1, roomStr.indexOf("卫")) + detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")).toInteger() + detail.toilet = roomStr.substring(roomStr.indexOf("卫") - 1, roomStr.indexOf("卫")).toInteger() } detail.direction = JQueryUtils.selectText("li:contains(房屋朝向)", doc).replace("房屋朝向", "") detail.fitment = JQueryUtils.selectText("li:contains(装修情况)", doc).replace("装修情况", "") @@ -420,11 +440,11 @@ public class LianjiaCrawlByChrome { def cityDoc = Jsoup.parse(driver.getPageSource()); def cityEles = JQueryUtils.select(".city_province a", cityDoc) def cityMap = [:] - def citys= ["长春","吉林"] + def citys= ["福州"] if(cityEles){ for(def e : cityEles) { def href = e.attr("href") - if (!href.contains("fang.lianjia") && citys.contains(e.text())){ + if (!href.contains("fang.lianjia")){ cityMap.put(e.text(), href) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy index acced0959..6a50bd201 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy @@ -4,9 +4,12 @@ import com.alibaba.fastjson.JSON import com.fangjia.core.util.DataUtil import com.fangjia.script.crawl.JQueryUtils import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject +import com.mongodb.Bytes import com.mongodb.DBCollection import com.mongodb.DBObject +import com.rabbitmq.client.impl.AMQImpl import org.jsoup.Jsoup import org.openqa.selenium.WebDriver import org.openqa.selenium.chrome.ChromeDriver @@ -19,9 +22,102 @@ class PaimaiTest { DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai") + public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + public static final DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl) + + static main(args) { def task = new PaimaiTest(); - task.run(); +// task.run(); +// task.stat() + task.updatePaimai() +// task.updateFieldPaimai() + } + + def updateFieldPaimai(){ + def q = new BasicDBObject() + q.put("city", new BasicDBObject('$in',["重庆","重庆市"])) + def index = 0 + def dbcursor = paimai43_COLL.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + for(def obj : dbcursor) { + index++ + println index + if(obj.address && obj.address.contains("可贷款")){ + obj.put("address", obj.address.substring(0,obj.address.indexOf("可贷款"))) + paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj)) + } + } + } + + def updatePaimai(){ + def q = new BasicDBObject() + q.put("url", Pattern.compile(".*jd.com.*")) + def dbcursor = paimaiResultColl.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def index =0 + for(def d : dbcursor){ + index++ + println index + if(!d.url.contains("jd.com")){ + continue + } + + if(!d.proince.contains("重庆")){ + continue + } + + println d.url + + def query = new BasicDBObject() + query.put("url", d.url) + def dbo = paimai43_COLL.findOne(query); + + DBObject obj = new BasicDBObject(); + d.each{k,v-> + obj.put(k,v) + } + + if(dbo){ + obj.m_date = new Date() + paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj)) + }else{ + obj.c_date = new Date() + obj.m_date = new Date() + try{ + paimai43_COLL.save(obj) + }catch(Exception e){ + println obj.url + } + } + } + } + + def stat(){ + def q = new BasicDBObject() + q.put("city", new BasicDBObject('$in', ["重庆","重庆市"])) + def dbcursor = paimaiResultColl.find(q) + def keys = [] + for(def db : dbcursor){ + if(db.matchCity){ + def key = db.matchCity +"##"+db.matchRegion+"##"+db.matchName; + if(!keys.contains(key)){ + keys << key + } +// try{ +// def jsonObject = OpenApiClient.addressMatch("重庆", db.address, "property", false, null, null) +// if(jsonObject != null && jsonObject.precise!=-1) { +// db.matchCity = jsonObject.get("city") +// db.matchRegion = jsonObject.get("district") +// db.matchName = jsonObject.get("name") +// paimaiResultColl.update(new BasicDBObject("_id", db._id),new BasicDBObject('$set',db)) +// println db.address+" "+db.matchName +// } +// }catch(Exception e){ +// println "error "+db.address +// } + + } + } + println keys.size() } def run(){ System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy index 8d971ee8a..926581423 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy @@ -11,14 +11,17 @@ import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject +import com.rabbitmq.client.AMQP import net.sf.json.JSONObject import org.apache.poi.hssf.usermodel.HSSFWorkbook import org.bson.types.ObjectId import java.text.DecimalFormat +import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicLong import java.util.regex.Matcher import java.util.regex.Pattern @@ -34,6 +37,9 @@ class CZDataBatch extends BaseBatch{ DBCollection DIST_COLL = MongoUtils.getCollectionByName("business", "jn_bank_district", "mongo.db.fangjia.com") //final def czIdMap = readCZDistrict("d:\\常州数据结果\\原始数据\\常州江南银行住宅编号.xls"); //final def czShopMap = readCZShop("d:\\常州数据结果\\原始数据\\常州江南银行商铺--更新ing(8).xls"); + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + static main(args) { runBatch(new CZDataBatch(), args); } @@ -41,11 +47,17 @@ class CZDataBatch extends BaseBatch{ @Override void process(Object args) { def citys = CommonDatautils.findAllCity(); + citys.remove("常州"); citys.add(0, "常州") for(def city : citys){ + + def weekend = DateUtils.getFirstDayOfWeek(new Date()) + def statMap = CommonDatautils.queryAllDistrictPriceByWeekend(city, weekend) + def commids = findAllJnBankDatas(city) + def districts = converList(CommonDatautils.findAllDistricts(city, "district", fields)); - districtRun(districts); + districtRun(districts, statMap, commids); def shops = CommonDatautils.findAllDistricts(city, "shop", fields); shops = mergeShop(shops); @@ -112,29 +124,48 @@ class CZDataBatch extends BaseBatch{ return rMap; } - def districtRun(def districts){ - + def districtRun(def districts, def statMap, commids){ def total = districts.size(); - def index = 0; - ExecutorService executorService = Executors.newFixedThreadPool(20) - + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(total) for(def i=0;i0){ + if(!room_num.toString().startsWith(house_floor.toString()) && room_num.toInteger()<=10){ + def newValue = house_floor +"-"+room_num + println "[${index}] "+room_num +" "+house_floor+" "+newValue + def udata = new BasicDBObject() + udata.put("room_num",newValue) + HOUSE_LOUDONG_COLL.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata), false, false); + } + } + } + }catch(Exception e){ + println d._id.toString() + } + + } } def export(){ def q = new BasicDBObject() @@ -157,29 +201,378 @@ class CqfdcMatch { return rets; } + def addHouseLoudong(){ + def q = new BasicDBObject() + def dbcursor = loudongConn.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def keys = [] + def index = 0 + for(def d : dbcursor) { + index++ + println index + if(d.room_num && (d.room_num.contains("商") || d.room_num.contains("社区用房"))){ + continue + } + if(d.area.toDouble()<=25){ + continue + } + def newRegion = RegionData.formatRegion(d.city, d.region) + if (newRegion) { + def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?") + def house_num = null + if (houseNumStr) { + house_num = houseNumStr.replace("号楼", "").replace("幢", "").replace("栋", "").replace("号", "") + } + if (house_num) { + def unitNum = DataHandleUtils.findMatchPattern(d.house_num, "\\d+单元") + if (unitNum) { + unitNum = unitNum.replace("单元", "") + } + def key = d.city + "##" + newRegion + "##" + d.district_name + "##" + house_num+ "##" + unitNum +"##"+d.room_num + if(!keys.contains(key) && d.room_num) { + keys << key + def count = findHouseLoudongDatas(d.city, newRegion, d.district_name, house_num, unitNum, d.room_num) + if(!count){ + def dist = null + if(d.match_city){ + dist = CommonDatautils.findDistrict(d.match_city, d.match_region, d.match_name,"district") + } + def udata = new BasicDBObject() + udata.put("city", d.city) + udata.put("region", newRegion) + udata.put("name", d.district_name) + udata.put("house_num", house_num) + if(unitNum){ + udata.put("house_num_unit", unitNum) + } + udata.put("room_num", d.room_num) + udata.put("house_area",d.area.toDouble()) + udata.put("status", 0) + udata.put("from", "房谱网") + if(d.floor_heigth && d.floor_heigth.isNumber()){ + udata.put("house_height", d.floor_heigth.toInteger()) + } + if(d.floor && d.floor.isNumber()){ + udata.put("house_floor", d.floor.toInteger()) + } + if(dist){ + udata.put("district_id", dist._id.toString()) + udata.put("name_in_seaweed", dist.name) + } + if(d.address){ + def address = AddressUtil.formatAddress(d.city, d.address) + if(address){ + udata.put("address", address) + } + } + udata.put("c_date", new Date()) + udata.put("m_date", new Date()) + HOUSE_LOUDONG_LINE_COLL.save(udata) + } + } + } + } + } + } + + def findHouseLoudongDatas(city, region, district_name, house_num, unitNum, room_num){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", district_name) + q.put("house_num", house_num) + if(unitNum){ + q.put("unit_num", unitNum) + } + q.put("room_num", room_num) + def count = HOUSE_LOUDONG_LINE_COLL.count(q) + return count; + } + def findAllLoundongByHouseNum(){ def q = new BasicDBObject() - def dbcursor = loudongConn.find(q).batchSize(5000) - def rets = [] - def map = [:] + def dbcursor = loudongConn.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(5000) + def keys = [] for(def d : dbcursor){ - def key = d.city +"##"+d.region+"##"+d.district_name+"##"+d.house_num - if(d.unit_num){ - key = key +"##"+d.unit_num - } - def detailMap= map.get(key) - if(!detailMap){ - detailMap = [:] - map.put(key,detailMap) - } + def newRegion = RegionData.formatRegion(d.city, d.region) + if(newRegion){ + def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?") + def house_num = null + if(houseNumStr){ + house_num = houseNumStr.replace("号楼","").replace("幢","").replace("栋","").replace("号","") + } + if(house_num){ + def unitNum = DataHandleUtils.findMatchPattern(d.house_num, "\\d+单元") + if(unitNum){ + unitNum = unitNum.replace("单元","") + } + def key = d.city +"##"+newRegion+"##"+d.district_name+"##"+house_num + if(unitNum){ + key = key +"##"+unitNum + } + if(!keys.contains(key)){ + keys << key + def obj = findLoudongOnLine(d.city, newRegion, d.district_name, house_num, d.unit_num) + println d.district_name+" "+d.house_num+"==${house_num}==${unitNum}" + if(obj){ + def udata = new BasicDBObject() + def areas = obj.areas + if(!areas.contains(d.area)){ + areas << d.area - if(detailMap.height){ + def forms = obj.from + if(!forms.contains("房谱网")){ + forms << "房谱网" + } + obj.m_date = new Date() + def query = new BasicDBObject("_id", obj._id) + LOUDONG_LINE_COLL.update(query, new BasicDBObject('$set', obj)) + + } + }else{ + def udata = new BasicDBObject() + udata.put("city", d.city) + udata.put("region", newRegion) + udata.put("name", d.district_name) + udata.put("house_num", house_num) + if(unitNum){ + udata.put("unit_num", unitNum) + } + if(d.address){ + def address = AddressUtil.formatAddress(d.city, d.address) + if(address){ + udata.put("address", address) + } + } + udata.put("areas",[d.area]) + udata.put("from",["房谱网"]) + if(d.floor_heigth && d.floor_heigth.toString().toInteger()){ + udata.put("floors", d.floor_heigth.toString().toInteger()) + } + + udata.put("c_date", new Date()) + udata.put("status", 1) + LOUDONG_LINE_COLL.save(udata) + } + } + } } } - return rets; } + def addFdcHouseLoudong(){ + def q = new BasicDBObject() + def dbcursor = loudongFdcConn.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(2000) + def keys = [] + def index = 0 + for(def d : dbcursor){ + index++ + println index + if(!d.use_type){ + continue + } + if(!d.use_type.contains("住宅")){ + continue + } + def newRegion = d.match_region + if(newRegion){ + def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num_desc, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?") + def house_num = null + if(houseNumStr){ + house_num = houseNumStr.replace("号楼","").replace("幢","").replace("栋","").replace("号","") + } + if(house_num){ + def key = d.city +"##"+newRegion+"##"+d.name+"##"+house_num+"##"+d.unit_num+"##"+d.room_num + if(!keys.contains(key)) { + keys << key + def count = findHouseLoudongDatas(d.city, newRegion, d.name, house_num, d.unit_num, d.room_num) + if(!count){ + def maxHeight = findMaxHeight(d.city, d.name, d.house_num_desc, d.unit_num) + def dist = null + if(d.match_city){ + dist = CommonDatautils.findDistrict(d.match_city, d.match_region, d.match_name,"district") + } + def floor = null + if(d.room_num && d.room_num.contains("-")) { + def arrs = d.room_num.split("-") + floor = arrs[0] + } + Integer room = null + Integer hall = null + if(d.room_type && d.room_type.contains("室")){ + def roomStr = d.room_type.substring(0,d.room_type.indexOf("室")) + if(roomStr){ + roomStr = roomStr.replace("一","1").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("六","6").replace("七","7").replace("八","8").replace("九","9").replace("十","10") + room = roomStr.toInteger() + } + + if(d.room_type.contains("厅")){ + def hallStr = d.room_type.substring(d.room_type.indexOf("室")+1, d.room_type.indexOf("厅")) + if(hallStr){ + hallStr = hallStr.replace("一","1").replace("两","2").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("六","6").replace("七","7").replace("八","8").replace("九","9").replace("十","10") + hall = hallStr.toInteger() + } + } + } + + + def udata = new BasicDBObject() + udata.put("city", d.city) + udata.put("region", newRegion) + udata.put("name", d.name) + udata.put("house_num", house_num) + if(d.unit_num){ + udata.put("house_num_unit", d.unit_num) + } + udata.put("room_num", d.room_num) + udata.put("house_area",d.area.toDouble()) + udata.put("status", 0) + udata.put("from", "重庆网上房地产") + udata.put("house_height", maxHeight) + if(room){ + udata.put("room", room) + } + if(hall){ + udata.put("hall", hall) + } + if(floor && floor.isNumber()){ + udata.put("house_floor", floor.toInteger()) + } + if(dist){ + udata.put("district_id", dist._id.toString()) + udata.put("name_in_seaweed", dist.name) + } + if(d.address){ + def address = AddressUtil.formatAddress(d.city, d.address) + if(address){ + udata.put("address", address) + } + } + udata.put("c_date", new Date()) + udata.put("m_date", new Date()) + HOUSE_LOUDONG_LINE_COLL.save(udata) + } + } + } + } + } + } + + public String getMatchString(String text,String pattern){ + Pattern p=Pattern.compile(pattern); + Matcher m=p.matcher(text); + if(m.find()){ + return m.group(0) + } + return null; + } + + def addFdcBaseLoudong(){ + def q = new BasicDBObject() + def dbcursor = loudongFdcConn.find(q).batchSize(5000) + def keys = [] + for(def d : dbcursor){ + if(!d.match_region){ + continue + } + def newRegion = d.match_region + + def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num_desc, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?") + def house_num = null + if(houseNumStr){ + house_num = houseNumStr.replace("号楼","").replace("幢","").replace("栋","").replace("号","") + } + if(house_num){ + def key = d.city +"##"+newRegion+"##"+d.name+"##"+house_num + if(d.unit_num) { + key = key + "##" + d.unit_num + + if (!keys.contains(key)) { + keys << key + def obj = findLoudongOnLine(d.city, newRegion, d.name, house_num, d.unit_num) + println d.name + " " + d.house_num_desc + "==${house_num}==${d.unit_num}" + + def maxHeight = findMaxHeight(d.city, d.name, d.house_num_desc, d.unit_num) + if (obj) { + + def areas = obj.areas + if (!areas.contains(d.area)) { + areas << d.area + + def forms = obj.from + if (!forms.contains("重庆网上房地产")) { + forms << "重庆网上房地产" + } + obj.m_date = new Date() + def query = new BasicDBObject("_id", obj._id) + LOUDONG_LINE_COLL.update(query, new BasicDBObject('$set', obj)) + + } + } else { + def udata = new BasicDBObject() + udata.put("city", d.city) + udata.put("region", newRegion) + udata.put("name", d.name) + udata.put("house_num", house_num) + if (d.unit_num) { + udata.put("unit_num", d.unit_num) + } + if (d.address) { + def address = AddressUtil.formatAddress(d.city, d.address) + if (address) { + udata.put("address", address) + } + } + udata.put("areas", [d.area]) + udata.put("from", ["重庆网上房地产"]) + udata.put("floors", maxHeight) + udata.put("c_date", new Date()) + udata.put("status", 1) + LOUDONG_LINE_COLL.save(udata) + } + } + } + } + } + } + + def findMaxHeight(city, name,house_num, unit_num){ + def q = new BasicDBObject() + q.put("city",city) + q.put("name",name) + q.put("house_num_desc",house_num) + q.put("unit_num",unit_num) + def dbcursor = loudongFdcConn.find(q) + def heights = [] + for(def h : dbcursor){ + if(h.nominal_floor && h.nominal_floor.isNumber()){ + def height = h.nominal_floor.toInteger() + if(!heights.contains(height)){ + heights << height + } + } + } + if(heights){ + heights = heights.sort() + return heights[heights.size()-1] + } + return null + } + + def findLoudongOnLine(city, region, name, house_num, unit_num){ + def q = new BasicDBObject() + q.put("city",city) + q.put("region",region) + q.put("name",name) + q.put("house_num",house_num) + if(unit_num){ + q.put("unit_num",unit_num) + } + def dbo = LOUDONG_LINE_COLL.findOne(q) + return dbo + } + + def stat(){ def q = new BasicDBObject() @@ -291,7 +684,51 @@ class CqfdcMatch { } def addLoudong(){ +// def stat = statBaseLoudong() +// println stat +// findAllLoundongByHouseNum(); + def stat1 = statHouseLoudong() + println stat1 +// addHouseLoudong() + } + def addFdcLoudong(){ +// addFdcBaseLoudong() + addFdcHouseLoudong() + } + + def statHouseLoudong(){ + def q = new BasicDBObject() + q.put("city", "重庆") + def dbcursor = HOUSE_LOUDONG_LINE_COLL.find(q) + def total = dbcursor.size() + def ids = [] + for(def d : dbcursor){ + if(d.district_id && !ids.contains(d.district_id)){ + ids << d.district_id + } + } + def map = [:] + map.rows_total = total + map.dist_count = ids.size() + return map + } + + def statBaseLoudong(){ + def q = new BasicDBObject() + q.put("city", "重庆") + def dbcursor = LOUDONG_LINE_COLL.find(q) + def total = dbcursor.size() + def ids = [] + for(def d : dbcursor){ + if(d.id_in_seaweed && !ids.contains(d.id_in_seaweed)){ + ids << d.id_in_seaweed + } + } + def map = [:] + map.rows_total = total + map.dist_count = ids.size() + return map } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/didata/MatchFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/didata/MatchFromExcel.groovy new file mode 100644 index 000000000..6cf7730ee --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/didata/MatchFromExcel.groovy @@ -0,0 +1,28 @@ +package com.fangjia.script.temp.shenjian.didata + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient + +/** + * 从excel中匹配出 + */ +ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\地数\\数据建设\\待审核小区(地图匹配).xlsx", { row -> + def city = row['A'] + def region = row['B'] + def district_name = row['C'] + + def gaodeRegion = row['H'] + def gaodeName = row['I'] + + def jsonObject = OpenApiClient.addressMatch(city, district_name, "district", false, region, district_name, "strict") + if (jsonObject != null && jsonObject.precise != -1) { + row['D'] = jsonObject.get("name") + } + + if (gaodeRegion && gaodeName) { + def jsonObject2 = OpenApiClient.addressMatch(city, gaodeName, "district", false, gaodeRegion, gaodeName, "strict") + if (jsonObject2 != null && jsonObject2.precise != -1) { + row['J'] = jsonObject2.get("name") + } + } +}); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/geo/BlockRegion.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/geo/BlockRegion.groovy new file mode 100644 index 000000000..bdd0d1110 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/geo/BlockRegion.groovy @@ -0,0 +1,46 @@ +package com.fangjia.script.temp.shenjian.geo + +import com.fangjia.script.gj.location.Point +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.xh.InsidePolygon +import com.mongodb.BasicDBList +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "block", "192.168.0.85", 28017) + +collection.find(new BasicDBObject()).forEach({ obj -> + if (obj.type == 'block') { + if (obj.region instanceof String) { + return; + } + BasicDBList reigon = (BasicDBList)obj.region + int count = 0 + List info = new ArrayList<>() + for (String s : reigon) { + info.add(s) + } + if (info.size() > 1) { + boolean find = false + for (String s : info) { + if (find) { + break + } + collection.find(new BasicDBObject("type", "region").append("name", s)).limit(1).forEach({obj2 -> + String border = obj2.border + String points = border.replaceAll(",", "-").replaceAll(";", "-") + List pointList = InsidePolygon.createPoint(points); + com.fangjia.script.looplineinfo.Point point = new com.fangjia.script.looplineinfo.Point(obj.lng, obj.lat) + if (InsidePolygon.aboutZone(point, pointList)) { + collection.update(new BasicDBObject("_id", obj._id), + new BasicDBObject('$set', new BasicDBObject("region", obj2.name))); + find = true; + } + }) + } + } else { + collection.update(new BasicDBObject("_id", obj._id), + new BasicDBObject('$set', new BasicDBObject("region", info.get(0)))); + } + } +}) \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromExcel.groovy index bd49875b0..235d779e4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromExcel.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromExcel.groovy @@ -2,6 +2,8 @@ package com.fangjia.script.temp.shenjian.poi import com.alibaba.fastjson.JSONObject import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.statistics.poi.AmapPoiDetail +import com.fangjia.script.temp.shenjian.util.CommonDtoUtils import com.fangjia.script.temp.shenjian.util.ExcelUtils import com.fangjia.script.temp.shenjian.util.GaoDeUtils import com.fangjia.script.temp.shenjian.util.HttpUtils @@ -14,13 +16,15 @@ import java.util.concurrent.atomic.AtomicLong class GetPoiFromExcel { - static final def URL = "http://192.168.0.216:38080/FangJiaDataService/saveAmapPoi" static final ExecutorService executorService = Executors.newFixedThreadPool(10) + static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoiDetail" + // 住宅类 + static final String typecodes = "120000|120100|120200|120201|120202|120203|120300|120301|120302|120303|120304"; static void main(String[] args) { AtomicLong atomicLong = new AtomicLong(0) //List fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Desktop\\房价网\\项目\\建行\\模型级联\\POI"), null, false); - List fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Desktop\\房价网\\项目\\建行\\模型级联\\高德反查无ID"), null, false); + List fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Documents\\知识库\\建行\\审核小区"), null, false); for (File file : fileList) { List> seaweeds = ExcelUtils.read(file.getPath()) for (List infoTemp : seaweeds) { @@ -31,9 +35,10 @@ class GetPoiFromExcel { String city = info.get(0) String region = info.get(1) String seaweedName = info.get(2) - List pois = GaoDeUtils.searchPoi(city, seaweedName, region, "120300|120301|120302"); + List pois = GaoDeUtils.searchPoi(city, seaweedName, region, typecodes); if (CollectionUtils.isNotEmpty(pois)) { - String jsonStr = JSONObject.toJSONString(pois) + List detailList = CommonDtoUtils.transformListV(AmapPoiDetail.class, pois); + String jsonStr = JSONObject.toJSONString(detailList) HttpUtils.postForJson(URL, jsonStr) atomicLong.incrementAndGet() } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy index 1a620af34..20def0a9b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy @@ -13,7 +13,13 @@ import com.fangjia.script.wenyan.auto.KeywordSearchUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject +import com.mongodb.client.model.Filters +import com.mongodb.client.model.UpdateOneModel +import com.mongodb.client.model.UpdateOptions +import com.mongodb.client.model.WriteModel import org.apache.commons.collections.CollectionUtils +import org.bson.Document +import org.bson.conversions.Bson import org.bson.types.ObjectId import java.util.concurrent.ExecutorService @@ -27,74 +33,85 @@ import java.util.concurrent.Executors */ class SeaweedAroundPoiSingal { - static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) + static def conn136 = MongoUtils.getCollectionByName3("fangjia", "seaweed_poi_sync", "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017") static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi" static void main(String[] args) { - def types = listAllType() - //def types = "0601" - DBObject query = new BasicDBObject() - query.put("status", 1) - conn136.find(query).batchSize(30).each {objTemp -> - final def obj = objTemp - def city = obj.city - def lng = obj.lng - def lat = obj.lat - List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") - if (CollectionUtils.isNotEmpty(pois)) { - String jsonStr = JSONObject.toJSONString(pois) - HttpUtils.postForJson(URL, jsonStr) - conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) - } else { - conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空"))) - } - println("已处理小区:" + obj) - } + getNewSeaweed() +// def types = listAllType() +// //def types = "0601" +// DBObject query = new BasicDBObject() +// query.put("status", 1) +// conn136.find(query).batchSize(30).each {objTemp -> +// final def obj = objTemp +// def city = obj.city +// def lng = obj.lng +// def lat = obj.lat +// List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") +// if (CollectionUtils.isNotEmpty(pois)) { +// String jsonStr = JSONObject.toJSONString(pois) +// HttpUtils.postForJson(URL, jsonStr) +// conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) +// } else { +// conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空"))) +// } +// println("已处理小区:" + obj) +// } } static getNewSeaweed() { DBObject query = new BasicDBObject() query.put("cat", "district") query.put("visible", 0) - //query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a")) DBObject filter = new BasicDBObject() filter.put("_id", 1) filter.put("city", 1) filter.put("region", 1) filter.put("name", 1) + filter.put("alias", 1) + filter.put("address", 1) + filter.put("addr_alias", 1) filter.put("lng", 1) filter.put("lat", 1) - List list = [] + List> writeModelList = new ArrayList<>() + def count = 0 DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj -> - if (conn136.count(new BasicDBObject("_id", obj._id)) > 0) { - // 已存在小区,无需同步 - } else { - DBObject newObj = new BasicDBObject() - newObj.put("_id", obj._id) - newObj.put("city", obj.city) - newObj.put("region", obj.region) - newObj.put("name", obj.name) - try { - newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) - newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) - } catch (Exception ex) { - print("经纬度为空:" + obj._id + " " + obj) - ex.printStackTrace() - } - newObj.put("status", 1) - newObj.put("flag", "未获取周边POI") + Document updateBson = new Document() + Document update = new Document() + update.put("city", obj.city) + update.put("region", obj.region) + update.put("name", obj.name) + update.put("alias", obj.alias) + update.put("address", obj.address) + update.put("addr_alias", obj.addr_alias) + try { + update.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + update.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + } + update.put("status", 1) + update.put("flag", "未精确获取高德信息") + updateBson.append('$set', update) - list << newObj - if (list.size() == 5000) { - conn136.insert(list) - list.clear() - } + Bson filterBson = Filters.eq("_id", obj._id) + UpdateOptions updateOptions = new UpdateOptions(); + updateOptions.upsert(true) + UpdateOneModel updateOneModel = new UpdateOneModel(filterBson, updateBson, updateOptions); + writeModelList.add(updateOneModel) + if (writeModelList.size() == 5000) { + conn136.bulkWrite(writeModelList) + count += 5000 + println(count) + writeModelList.clear() } } - conn136.insert(list) - list.clear() + if (writeModelList.size() > 0) { + conn136.bulkWrite(writeModelList) + writeModelList.clear() + } } static def getByIdAndInsert(String id) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy new file mode 100644 index 000000000..ea6d9cc67 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy @@ -0,0 +1,163 @@ +package com.fangjia.script.temp.shenjian.poi + + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.statistics.poi.AmapPoiDetail +import com.fangjia.script.temp.shenjian.util.CommonDtoUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.wenyan.auto.KeywordSearchUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import com.mongodb.client.model.Filters +import com.mongodb.client.model.UpdateOneModel +import com.mongodb.client.model.UpdateOptions +import com.mongodb.client.model.WriteModel +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang.StringUtils +import org.bson.Document +import org.bson.conversions.Bson +import org.bson.types.ObjectId + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区POI信息精确查找 + * + * @author shenjian + * @since 2022/5/25 + */ +class SeaweedPoiDetail { + + static def conn136 = MongoUtils.getCollectionByName3("fangjia", "seaweed_poi_sync", "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017") + static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoiDetail" + // 住宅类 + static final String typecodes = "120000|120100|120200|120201|120202|120203|120300|120301|120302|120303|120304"; + static final ExecutorService executorService = Executors.newFixedThreadPool(20) + + static void main(String[] args) { + DBObject query = new BasicDBObject() + query.put("status", 1) + conn136.find(query).batchSize(1000).each {objTemp -> + final def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + def city = obj.city + def region = obj.region + def name = obj.name + def alias = obj.alias + def address = obj.address + def addr_alias = obj.addr_alias + StringJoiner stringJoiner = new StringJoiner("|") + // 主小区名搜索 + stringJoiner.add(name) + // 根据别名搜 + if (alias != null && CollectionUtils.isNotEmpty(alias)) { + for (String str : alias) { + stringJoiner.add(str) + } + } + // 根据路名搜 + if (StringUtils.isNotBlank(address)) { + stringJoiner.add(address) + } + // 根据路别名搜 + if (addr_alias != null && CollectionUtils.isNotEmpty(addr_alias)) { + for (String str : addr_alias) { + stringJoiner.add(str) + } + } + + String keywords = stringJoiner.toString() + List pois = GaoDeUtils.searchPoi(city, keywords, region, typecodes) + if (CollectionUtils.isEmpty(pois)) { + pois = GaoDeUtils.searchPoi(city, name, region, typecodes) + } + if (CollectionUtils.isEmpty(pois)) { + pois = GaoDeUtils.searchPoi(city, address, region, typecodes) + } + + List detailList = CommonDtoUtils.transformListV(AmapPoiDetail.class, pois); +// for (AmapPoi amapPoi : pois) { +// List amapPoiList = GaoDeUtils.getAmapDetailById(amapPoi.getId()); +// if (CollectionUtils.isNotEmpty(amapPoiList)) { +// detailList.addAll(amapPoiList) +// } +// } + if (CollectionUtils.isNotEmpty(detailList)) { + String jsonStr = JSONObject.toJSONString(detailList) + HttpUtils.postForJson(URL, jsonStr) + conn136.updateOne(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取到"))) + } else { + conn136.updateOne(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "未获取到小区"))) + } + println("已处理小区:" + obj) + } + }) + } + executorService.shutdown() + } + + static getNewSeaweed() { + DBObject query = new BasicDBObject() + query.put("cat", "district") + query.put("visible", 0) + + DBObject filter = new BasicDBObject() + filter.put("_id", 1) + filter.put("city", 1) + filter.put("region", 1) + filter.put("name", 1) + filter.put("alias", 1) + filter.put("address", 1) + filter.put("addr_alias", 1) + filter.put("lng", 1) + filter.put("lat", 1) + + List> writeModelList = new ArrayList<>() + def count = 0 + DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj -> + Document updateBson = new Document() + Document update = new Document() + update.put("city", obj.city) + update.put("region", obj.region) + update.put("name", obj.name) + update.put("alias", obj.alias) + update.put("address", obj.address) + update.put("addr_alias", obj.addr_alias) + try { + update.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + update.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + } + update.put("status", 1) + update.put("flag", "未精确获取高德信息") + updateBson.append('$set', update) + + Bson filterBson = Filters.eq("_id", obj._id) + UpdateOptions updateOptions = new UpdateOptions(); + updateOptions.upsert(true) + UpdateOneModel updateOneModel = new UpdateOneModel(filterBson, updateBson, updateOptions); + writeModelList.add(updateOneModel) + if (writeModelList.size() == 5000) { + conn136.bulkWrite(writeModelList) + count += 5000 + println(count) + writeModelList.clear() + } + } + if (writeModelList.size() > 0) { + conn136.bulkWrite(writeModelList) + writeModelList.clear() + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/CommonDtoUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/CommonDtoUtils.groovy new file mode 100644 index 000000000..89dda4684 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/CommonDtoUtils.groovy @@ -0,0 +1,40 @@ +package com.fangjia.script.temp.shenjian.util; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; + +/** + * 通用实体与DTO互相转换工具类 + * + * @author shenjian + * @date 2021/06/16 + */ +public class CommonDtoUtils { + + public static T transformV(Class targetClass, Object source) { + if (source == null) { + return null; + } else { + try { + String jsonSource = JSON.toJSONString(source); + return JSONObject.parseObject(jsonSource, targetClass); + } catch (Exception var3) { + throw var3; + } + } + } + + public static List transformListV(Class targetClass, List listSource) { + if (listSource == null) { + return null; + } else { + try { + String jsonSource = JSON.toJSONString(listSource); + return JSONArray.parseArray(jsonSource, targetClass); + } catch (Exception var3) { + throw var3; + } + } + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 49c7c18de..6b880c5ea 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -7,10 +7,12 @@ import com.fangjia.core.util.DataUtil import com.fangjia.core.util.HttpRequest import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.statistics.poi.AmapPoiDetail import com.fangjia.script.util.GeoHelper import com.fangjia.script.util.HttpRequestUtil import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import org.apache.commons.lang.StringUtils +import org.openqa.selenium.json.Json import java.util.concurrent.atomic.AtomicInteger @@ -88,12 +90,25 @@ class GaoDeUtils { if (json && json.info && json.info == "OK") { count = json.getString("count").toInteger() result.addAll(DataUtil.toMongoDbObj(json.pois)) - } else if (json && json.infocode && json.infocode == 10003) { - throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") - } else if (json && json.infocode && json.infocode == 10004) { + } else if (json && json.infocode && json.infocode == "10044") { + synchronized (this) { + for (int i = 0; i < GAO_DE_AK.size(); i++) { + if (GAO_DE_AK[i] == params.key) { + if (i + 1 >= GAO_DE_AK.size()) { + System.exit(1) + } + keyUsed = new AtomicInteger(i+1) + } + } + break; + } + } else if (json && json.infocode && json.infocode == "10004") { // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 Thread.sleep(60000) break; + } else if (json && json.infocode && json.infocode == "10017") { + // 所请求的资源不可用 + System.exit(1) } else { break } @@ -146,7 +161,7 @@ class GaoDeUtils { synchronized (this) { for (int i = 0; i < GAO_DE_AK.size(); i++) { if (GAO_DE_AK[i] == params.key) { - if (i + 1 > GAO_DE_AK.size()) { + if (i + 1 >= GAO_DE_AK.size()) { System.exit(1) } keyUsed = new AtomicInteger(i+1) @@ -160,7 +175,9 @@ class GaoDeUtils { break; } else if (json && json.infocode && json.infocode == "10017") { // 所请求的资源不可用 - System.exit(1) + //System.exit(1) + Thread.sleep(60000) + break; } else { break } @@ -222,12 +239,65 @@ class GaoDeUtils { } catch(Exception ex) { ex.printStackTrace() } + } else if (json && json.infocode && json.infocode == "10044") { + synchronized (this) { + for (int i = 0; i < GAO_DE_AK.size(); i++) { + if (GAO_DE_AK[i] == params.key) { + if (i + 1 >= GAO_DE_AK.size()) { + System.exit(1) + } + keyUsed = new AtomicInteger(i+1) + } + } + } } else if (json && json.infocode && json.infocode == "10004") { // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 Thread.sleep(60000) } else if (json && json.infocode && json.infocode == "10017") { // 所请求的资源不可用 - System.exit(1) + //System.exit(1) + Thread.sleep(60000) + } + return Collections.emptyList() + } + + /** + * 获取详情 + */ + static List getAmapDetailById(id){ + def params = [:] + params.key = GAO_DE_AK[keyUsed.intValue()] + params.id = id + params.output = 'json' + JSONObject json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v3/place/detail", params); + if (json == null) { + json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v3/place/detail", params); + } + if (json && json.info && json.info == "OK") { + try { + List ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class) + return ret + } catch(Exception ex) { + ex.printStackTrace() + } + } else if (json && json.infocode && json.infocode == "10044") { + synchronized (this) { + for (int i = 0; i < GAO_DE_AK.size(); i++) { + if (GAO_DE_AK[i] == params.key) { + if (i + 1 >= GAO_DE_AK.size()) { + System.exit(1) + } + keyUsed = new AtomicInteger(i+1) + } + } + } + } else if (json && json.infocode && json.infocode == "10004") { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + } else if (json && json.infocode && json.infocode == "10017") { + // 所请求的资源不可用 + //System.exit(1) + Thread.sleep(60000) } return Collections.emptyList() } @@ -270,7 +340,7 @@ class GaoDeUtils { synchronized (this) { for (int i = 0; i < GAO_DE_AK.size(); i++) { if (GAO_DE_AK[i] == params.key) { - if (i + 1 > GAO_DE_AK.size()) { + if (i + 1 >= GAO_DE_AK.size()) { System.exit(1) } keyUsed = new AtomicInteger(i+1) @@ -278,13 +348,13 @@ class GaoDeUtils { } break; } - } else if (json && json.infocode && json.infocode == "10004") { + } else if (json && json.infocode && (json.infocode == "10004" || json.infocode == "10021" || json.infocode == "10017")) { // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 Thread.sleep(60000) break; - } else if (json && json.infocode && json.infocode == "10017") { - // 所请求的资源不可用 - System.exit(1) + } else if (json && json.infocode && (json.infocode == "20000")) { + // 无效参数 + break; } } return result @@ -326,7 +396,7 @@ class GaoDeUtils { synchronized (this) { for (int i = 0; i < GAO_DE_AK.size(); i++) { if (GAO_DE_AK[i] == params.key) { - if (i + 1 > GAO_DE_AK.size()) { + if (i + 1 >= GAO_DE_AK.size()) { System.exit(1) } keyUsed = new AtomicInteger(i+1) @@ -367,7 +437,16 @@ class GaoDeUtils { return ["lng": arrs[0], "lat": arrs[1]] } } else if (json && json.infocode && json.infocode == "10044") { - System.exit(1) + synchronized (this) { + for (int i = 0; i < GAO_DE_AK.size(); i++) { + if (GAO_DE_AK[i] == params.key) { + if (i + 1 >= GAO_DE_AK.size()) { + System.exit(1) + } + keyUsed = new AtomicInteger(i+1) + } + } + } } else if (json && json.infocode && json.infocode == "10004") { // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 Thread.sleep(60000) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy index 668172385..c17285a78 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy @@ -128,10 +128,10 @@ class SeaweedUtils { */ static formatHouseholdCount(def household_count) { if (household_count && FieldValidatorUtils.checkValue(household_count) ) { - if (Long.valueOf(household_count) < 50) { + if (Integer.valueOf(household_count) < 50) { return "" // 户数 } - return household_count + return household_count.toInteger() } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index 1d80f9b2a..9a9fd90ad 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -514,6 +514,7 @@ class DataStandardBatch extends BaseBatch{ def filterSpacialChar(def s){ def pattern = ~/[()!;\·]\[【】()]/ def matcher = pattern.matcher(s) + def count = matcher.getCount() if(count && count>0){ return matcher.replaceAll('_') diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index 97af5b6dc..de2db639a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -43,8 +43,8 @@ class DataStat { def source = "58同城"// def citys = findCrawlCity(source, 202202); for(def city : citys){ - def count1 = findCount(source, city, 202202); - def count2 = findCount(source, city, 202203); + def count1 = findCount(source, city, 202203); + def count2 = findCount(source, city, 202204); println city +" "+count1+" "+count2 // println count1 // println "--- ----" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index ee0977a28..ed1b90eba 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -40,13 +40,13 @@ class Esf58Crawl extends BaseBatch{ def static useDistKeys = [] static def totalCount = new AtomicLong() - def cityList = ["成都":"cd|102","杭州":"hz|79","重庆":"cq|37","苏州":"su|5","无锡":"wx|93", "深圳":"sz|4","郑州":"zz|342", "长沙":"cs|414","东莞":"dg|413", + def cityList = ["福州":"fz|304","成都":"cd|102","杭州":"hz|79","重庆":"cq|37","苏州":"su|5","无锡":"wx|93", "深圳":"sz|4","郑州":"zz|342", "长沙":"cs|414","东莞":"dg|413", "保定":"bd|424","保亭":"baoting|10367","南宁":"nn|845", "扬州":"yz|637","汕头":"st|783","泰州":"taizhou|693","淮安":"ha|968","湛江":"zhanjiang|791","锦州":"jinzhou|2354","昆明":"km|541","柳州":"liuzhou|7133","肇庆":"zq|901","连云港":"lyg|2049","日照":"rizhao|3177","常德":"changde|872", "沧州":"cangzhou|652", "泉州":"qz|291","清远":"qingyuan|7303", "西宁":"xn|2052","遵义":"zunyi|7620","株洲":"zhuzhou|1086","贵阳":"gy|2015","岳阳":"yy|821","徐州":"xz|471", "哈尔滨":"hrb|202","宜昌":"yc|858", "廊坊":"lf|772","秦皇岛":"qhd|1078","惠州":"huizhou|722","洛阳":"luoyang|556", "石家庄":"sjz|241","太原":"ty|740","盐城":"yancheng|613","信阳":"xy|8694","南充":"nanchong|2378","南阳":"ny|592","厦门":"xm|606", "绵阳":"mianyang|1057","济宁":"jining|450","邯郸":"hd|572", "淮北":"huaibei|9357","滁州":"chuzhou|10266", - "福州":"fz|304","蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158", + "蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158", "兰州":"lz|952","合肥":"hf|837", "乌鲁木齐":"xj|984","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","潍坊":"wf|362", "芜湖":"wuhu|2045","东莞":"dg|413","常州":"cz|463","佛山":"fs|222", "南通":"nt|394","吉林":"jl|700","中山":"zs|771","阜阳":"fy|2325", @@ -192,7 +192,8 @@ class Esf58Crawl extends BaseBatch{ void process(Object args) { //addContent("福州","https://fz.58.com/xiaoqu/", 70); - def citys = ["北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏"] +// "宁波","佛山","南通","烟台","临沂","呼和浩特","海口","广州", + def citys = ["唐山","泰安","襄阳","台州","镇江","银川","江门","荆州","北京","衡阳","南京","滨州","张家口","商丘","赣州","湖州","安庆","聊城","枣庄","阜阳","西安","芜湖","吉林","遂宁","鞍山","东营","安阳","咸阳"] for(def city : citys){ def str = cityList.get(city) for(def i=0;i<1;i++){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index fec88b8c5..b620f3098 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -18,7 +18,7 @@ class SynchDistrictPrice { // def sourceMap = ["fang.com":"搜房","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房", // "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"] //"fang.com":"搜房", - def sourceMap = ["fang.com":"搜房","ke.com":"贝壳找房","5i5j.com":"我爱我家"] + def sourceMap = ["ke.com":"贝壳找房","5i5j.com":"我爱我家"] static main(args) { def task = new SynchDistrictPrice() task.run() @@ -26,7 +26,7 @@ class SynchDistrictPrice { def run(){ - def citys = CommonUtils.findAllCity() + def citys = ["重庆"]//CommonUtils.findAllCity() sourceMap.each { k, v -> for(def city : citys){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index bd0fbb3f3..4866f25b8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] for(def source :sources){ - def count1 = findTotalCount(source, 202202); - def count2 = findTotalCount(source, 202203); + def count1 = findTotalCount(source, 202203); + def count2 = findTotalCount(source, 202204); println source +" "+count1 +" "+count2 } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy index fe433f087..21887e3db 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy @@ -169,7 +169,7 @@ public class CommonUtils { static void sendMessage(String queueName, byte[] bytes){ ConnectionFactory factory = new ConnectionFactory(); - factory.setUri("amqp://guest:guest@192.168.0.19:45672/%2f"); + factory.setUri("amqp://guest:guest@192.168.0.100:35677/%2f"); Connection conn = factory.newConnection(); Channel channel = conn.createChannel(); channel.queueDeclare(queueName, true, false, false, null); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy index 69a75d454..7e6e73263 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy @@ -92,11 +92,6 @@ public class DbUtils{ } } - - public static void executeQueryOnSeawater(BasicDBObject query,Closure c, BasicDBObject fields = null){ - seawater.find(query,fields).batchSize(10000).each(c) - } - public static void executeQueryOnSeaweed(BasicDBObject query,Closure c, BasicDBObject fields = null){ SEAWEED_SLAVE.find(query,fields).batchSize(1000).each(c) } @@ -171,10 +166,6 @@ public class DbUtils{ return SEAWEED_SLAVE.findOne(query,fields); } - public static int executeCountOnSeawater(BasicDBObject query){ - return seawater.count(query); - } - public static BasicDBObject buildSeaweedQuery(def city = null, def region = null, def name = null, def cat = "district", def visible = 0){ BasicDBObject query = new BasicDBObject() if(city){ @@ -387,31 +378,6 @@ public class DbUtils{ return district.lat && district.lng && district.lng > 0 && district.lat > 0 } -/** - * 记录修改小区库的操作日志 - * @param city - * @param region - * @param name - * @return - */ - def static saveDeleteLog(String city, String region,String name,String category,int status){ - if(status!=1){ - return - } - DBObject obj = new BasicDBObject() - obj.put("city", city); - obj.put("region", region); - obj.put("district_name", name); - obj.put("category" , category) - obj.put("origin" , "脚本") - if(status==1){ - obj.put("type" , "remove") - } - obj.put("c_date" , new Date()) - obj.put("m_date" , new Date()) - OPERATION_LOG.save(obj) - } - public static DBCollection getCollection(String host, int port, String dbName ,String collName){ def collection = COLL_CACHE["${host}#${port}#${dbName}#${collName}"] if(!collection){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index 5ef52da25..3d4e855f2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -229,6 +229,153 @@ public class OpenApiClient { return null } + public static JSONObject searchLoudong(String id, String houseNum, String unitNum){ + try{ + Map params = new HashMap(); + params.put("id", id); + params.put("houseNumber", houseNum); + params.put("unitNumber", unitNum); + params.put("token", TOKEN); + String content = new HttpRequest().get("http://open.fangjia.com/property/searchLoudong", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + while (result == null) { + result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/property/searchLoudong", params, "utf-8") + println("${id} ${houseNum} 程序调用异常") + Thread.sleep(6000) + } + if(result){ + if (result.getInt("code") == 200) { + return result; + } else if (result.getInt("code") !=0) { + println("${id} ${houseNum}") + } + } + + } catch (Exception e) { + println("${id} ${houseNum} ${e.getMessage()}") + e.printStackTrace(); + } + return null + } + + public static JSONObject searchHouseLoudong(String id, String houseNum, String unitNum, String roomNum){ + try{ + Map params = new HashMap(); + params.put("id", id); + params.put("houseNumber", houseNum); + params.put("unitNumber", unitNum); + params.put("roomNumber", roomNum); + params.put("token", TOKEN); + String content = new HttpRequest().get("http://localhost:8080/fangjia-open/property/searchHouseLoudong", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + while (result == null) { + result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/property/searchHouseLoudong", params, "utf-8") + println("${id} ${houseNum} 程序调用异常") + Thread.sleep(6000) + } + if(result){ + if (result.getInt("code") == 200) { + return result; + } else if (result.getInt("code") !=0) { + println("${id} ${houseNum}") + } + } + + } catch (Exception e) { + println("${id} ${houseNum} ${e.getMessage()}") + e.printStackTrace(); + } + return null + } + + + public static JSONObject webankEvaluate(String city, String region, String address, String area){ + try{ + Map params = new HashMap(); + params.put("city", city); + params.put("district", region); + params.put("address", address); + params.put("area", area.toString()); + params.put("token", TOKEN); + String content = new HttpRequest().get("http://open.fangjia.com/webank/evaluate", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + while (result == null) { + result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/webank/evaluate", params, "utf-8") + println("${city} ${address} 程序调用异常") + Thread.sleep(6000) + } + if(result){ + if (result.getInt("code") == 200) { + return result; + } else if (result.getInt("code") !=0) { + println("${city} ${address} 程序调用异常") + } + } + + } catch (Exception e) { + println("${city} ${address} ${e.getMessage()}") + e.printStackTrace(); + } + return null + } + + + public static JSONObject standard2Action(String city, String address) { + try { + Map params = new HashMap(); + params.put("city", city); + params.put("address", address); + params.put("token", TOKEN); + params.put("__refresh_cache","1") + + String content = new HttpRequest().get("https://open.fangjia.com/v2/address/standard", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + while (result == null) { + result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/v2/address/standard", params, "utf-8") + println("${city} ${address} 程序调用异常") + Thread.sleep(6000) + } + if (result.getInt("code") == 0) { + return result.getJSONObject("result"); + } else if (result.getInt("code") !=0) { + println("${city} ${address}") + } + } catch (Exception e) { + println("${city} ${address} ${e.getMessage()}") + e.printStackTrace(); + } + return null + } + + + public static JSONObject standard1Action(String city, String address) { + try { + Map params = new HashMap(); + params.put("city", city); + params.put("address", address); + params.put("token", TOKEN); + params.put("__refresh_cache","1") + + String content = new HttpRequest().get("https://open.fangjia.com/address/standard", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + while (result == null) { + result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/address/standard", params, "utf-8") + println("${city} ${address} 程序调用异常") + Thread.sleep(6000) + } + if (result.getInt("code") == 200) { + return result.getJSONObject("result"); + } else if (result.getInt("code") !=0) { + println("${city} ${address}") + } + } catch (Exception e) { + println("${city} ${address} ${e.getMessage()}") + e.printStackTrace(); + } + return null + } + + public static JSONObject standardize(String city, String address) { try { Map params = new HashMap(); @@ -593,6 +740,91 @@ public class OpenApiClient { } + static JSONObject facilitys(String city, String region, String name) { + Map params = new HashMap(); + params.put("token", TOKEN); + params.put("city", city) + params.put("region", region) + params.put("name", name) + try { + String content = new HttpRequest().get("http://open.fangjia.com/district/facilitys", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + + return result + } catch (Exception e) { + + } + return null + } + + static JSONObject searchPaimais(String city, String region, String name, String startDate,limit) { + Map params = new HashMap(); + params.put("token", TOKEN); + params.put("city", city) + params.put("district", region) + params.put("name", name) + params.put("startDate", startDate) + params.put("limit", "100") + try { + String content = new HttpRequest().get("http://open.fangjia.com/auction/searchPaimais", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + return result + } catch (Exception e) { + + } + return null + } + + + static JSONObject searchDealPrice(String id, String startDate, limit) { + Map params = new HashMap(); + params.put("token", TOKEN); + params.put("id", id) + params.put("startDate", startDate) + params.put("limit", limit.toString()) + try { + String content = new HttpRequest().get("http://open.fangjia.com/dealPrice/searchList", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + return result + } catch (Exception e) { + + } + return null + } + + static JSONObject houseList(String id, String startDate, limit) { + Map params = new HashMap(); + params.put("token", TOKEN); + params.put("id", id) + params.put("startTime", startDate) + params.put("pageSize", limit.toString()) + try { + String content = new HttpRequest().get("http://open.fangjia.com/house/list", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + return result + } catch (Exception e) { + + } + return null + } + + + static JSONObject evaluaionList(String id, String startDate, limit) { + Map params = new HashMap(); + params.put("token", TOKEN); + params.put("id", id) + params.put("startDate", startDate) + params.put("limit", limit.toString()) + try { + String content = new HttpRequest().get("http://open.fangjia.com/evaluation/bsList", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + return result + } catch (Exception e) { + + } + return null + } + static JSONObject districtDetail(String city, String region, String name) { Map params = new HashMap(); params.put("token", TOKEN); diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoiDetail.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoiDetail.java new file mode 100644 index 000000000..8192b359c --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoiDetail.java @@ -0,0 +1,71 @@ +package com.fangjia.script.statistics.poi; + +import java.util.List; + +/** + * @author shenjian + * @since 2022/5/25 + */ +@lombok.NoArgsConstructor +@lombok.Data +public class AmapPoiDetail { + + @com.alibaba.fastjson.annotation.JSONField(name = "id") + private String id; + @com.alibaba.fastjson.annotation.JSONField(name = "parent") + private List parent; + @com.alibaba.fastjson.annotation.JSONField(name = "childtype") + private List childtype; + @com.alibaba.fastjson.annotation.JSONField(name = "name") + private String name; + @com.alibaba.fastjson.annotation.JSONField(name = "type") + private String type; + @com.alibaba.fastjson.annotation.JSONField(name = "typecode") + private String typecode; + @com.alibaba.fastjson.annotation.JSONField(name = "biz_type") + private List bizType; + @com.alibaba.fastjson.annotation.JSONField(name = "address") + private String address; + @com.alibaba.fastjson.annotation.JSONField(name = "location") + private String location; + @com.alibaba.fastjson.annotation.JSONField(name = "tel") + private String tel; + @com.alibaba.fastjson.annotation.JSONField(name = "pname") + private String pname; + @com.alibaba.fastjson.annotation.JSONField(name = "cityname") + private String cityname; + @com.alibaba.fastjson.annotation.JSONField(name = "adname") + private String adname; + @com.alibaba.fastjson.annotation.JSONField(name = "importance") + private List importance; + @com.alibaba.fastjson.annotation.JSONField(name = "shopid") + private List shopid; + @com.alibaba.fastjson.annotation.JSONField(name = "shopinfo") + private String shopinfo; + @com.alibaba.fastjson.annotation.JSONField(name = "poiweight") + private List poiweight; + @com.alibaba.fastjson.annotation.JSONField(name = "distance") + private String distance; + @com.alibaba.fastjson.annotation.JSONField(name = "biz_ext") + private BizExtDTO bizExt; + @com.alibaba.fastjson.annotation.JSONField(name = "photos") + private List photos; + + @lombok.NoArgsConstructor + @lombok.Data + public static class BizExtDTO { + @com.alibaba.fastjson.annotation.JSONField(name = "rating") + private List rating; + @com.alibaba.fastjson.annotation.JSONField(name = "cost") + private String cost; + } + + @lombok.NoArgsConstructor + @lombok.Data + public static class PhotosDTO { + @com.alibaba.fastjson.annotation.JSONField(name = "title") + private List title; + @com.alibaba.fastjson.annotation.JSONField(name = "url") + private String url; + } +} diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java index 46545fc04..15909a4c9 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java @@ -1,12 +1,12 @@ package com.fangjia.service.address; import com.fangjia.core.RegionData; -import com.fangjia.core.util.*; +import com.fangjia.core.util.DataUtil; +import com.fangjia.core.util.GeoUtils; import com.fangjia.dal.QueryParam; import com.fangjia.dal.common.DistrictKey; import com.fangjia.dal.common.ServerAreaData; import com.fangjia.dal.mongo.fangjia.model.ServerArea; -import com.fangjia.dal.mongo.model.AmapPoi; import com.fangjia.dal.mongo.model.bkoff.HouseLoudong; import com.fangjia.dal.mongo.model.bkoff.Loudong; import com.fangjia.dal.seaweed.dao.SeaweedDao; @@ -16,17 +16,15 @@ import com.fangjia.service.address.processor.AddressDistrictProcessors; import com.fangjia.service.address.processor.AddressProcessors; import com.fangjia.service.api.AliyunService; import com.fangjia.service.bo.AliyunBo; -import com.fangjia.service.bo.MatchAddressScore; import com.fangjia.service.datastore.DistrictService; import com.fangjia.service.mongo.addrepo.AddressRepoService; import com.fangjia.service.mongo.bkoff.HouseLoudongService; import com.fangjia.service.mongo.bkoff.LoudongService; import com.google.common.base.Function; import com.google.common.collect.MapMaker; -import net.sf.json.JSONObject; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; -import org.bson.types.ObjectId; + import java.util.*; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; @@ -759,6 +757,13 @@ public class AddressParserServiceImpl implements AddressParserService { double simScore; boolean onSameRoad = false; boolean sameDistrict = isSameDistrict(address, metaData); + if(PreciseLevel.STRICT == address.getPreciseLevel() && !sameDistrict){ + return 0; + } +// //当地图解析出来的区域和用户地址中的区域一致时,只匹配指定区域下面的小区 +// if(address.getGeoRegion() != null && address.getGeoRegion().equals(address.getDistrict()) && !sameDistrict){ +// return 0; +// } Iterator> addrEntrySet = metaData.getAddrePinYinMap().entrySet().iterator(); Map.Entry addrEntry; while(addrEntrySet.hasNext()){ @@ -808,7 +813,6 @@ public class AddressParserServiceImpl implements AddressParserService { } } } - String keywordPinYin = address.getKeywordsPinYin(); String keywords = address.getKeywords(); String fuzzyKeywordsPinYin = address.getFuzzyKeywordsPinYin(); @@ -892,7 +896,7 @@ public class AddressParserServiceImpl implements AddressParserService { if(name.length() == 2 && (name.equals(address.getOrgAddress()) || name.equals(address.getEffectiveAddress()))){ keywordScore = Math.max(keywordScore, namePriority); }else if(name.length() > 3 && (address.getOrgAddrePinYin().contains(namePinYin) || enhanceContains(address.getOrgAddress(), name))){ - double similar = AddressParserUtil.getSimilarityRatio(name, address.getEffectiveAddress()); + double similar = AddressParserUtil.getSimilarityRatio(name, address.getOrgAddress()); if(similar == 1){ keywordScore = Math.max(similar, keywordScore) * namePriority * 1.0001; }else if(similar > 0.1){ @@ -907,20 +911,27 @@ public class AddressParserServiceImpl implements AddressParserService { if (standardAddressScore > 0 && keywordScore > 0) { matchLevelScore = 0.9; similarityScore = Math.max(standardAddressScore, keywordScore) * SIMILARITY_RATE; - } else if(keywordScore > 0 && streetScore > 0){ - matchLevelScore = 0.8; - similarityScore = Math.max(streetScore, keywordScore * 0.8) * SIMILARITY_RATE; } else if (standardAddressScore > 0) { - matchLevelScore = 0.7; + matchLevelScore = 0.8; similarityScore = standardAddressScore * SIMILARITY_RATE; + } else if(keywordScore > 0 && streetScore > 0){ + matchLevelScore = 0.7; + similarityScore = Math.max(streetScore, keywordScore * 0.8) * SIMILARITY_RATE; } else if (keywordScore > 0) { matchLevelScore = 0.6; similarityScore = keywordScore * SIMILARITY_RATE; } else if (fuzzyKeywordsScore > 0) { - matchLevelScore = 0.5; + if(address.hasCoordinate() && metaData.getLng() != null && GeoUtils.calculateDistance(address.getLng(), address.getLat(), + metaData.getLng(), metaData.getLat()) < 100){ + matchLevelScore = 0.7; + }else{ + matchLevelScore = 0.4; + } similarityScore = fuzzyKeywordsScore * SIMILARITY_RATE; + }else if(metaData.getId().equals(address.getPolygonSeaweedId())){ + matchLevelScore = 0.4; + similarityScore = 0.05; } - if (similarityScore > 0) { if(metaData.getBuildingType() != null){ if ("shop".equals(metaData.getBuildingType().toString())) { @@ -942,10 +953,6 @@ public class AddressParserServiceImpl implements AddressParserService { * similarityScore = 相似度部分(0.07) + 类型部分(0.01) < 0.1 * */ double score = matchLevelScore + similarityScore; - if(address.hasCoordinate() && metaData.hasCoordinate() - && GeoUtils.calculateDistance(address.getLng(), address.getLat(), metaData.getLng(), metaData.getLat()) > 3 * 1000){ - score = 0d; - } return score; } @@ -980,7 +987,7 @@ public class AddressParserServiceImpl implements AddressParserService { return false; } if(StringUtils.isBlank(address.getDistrict()) && StringUtils.isBlank(address.getSuggestDistrict())){ - return false; + return true; } String district = metaData.getDistrict(); return district.equals(address.getDistrict()) || district.equals(address.getSuggestDistrict()); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java index 720da51bb..ed723c1d5 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java @@ -61,6 +61,8 @@ public class Address { private String suggestDistrict; private String suggestName; private String suggestNamePinYin; + private String polygonSeaweedId; + private String geoRegion; private List buildingNumbers; private PreciseLevel preciseLevel; private boolean pinYin; @@ -750,6 +752,14 @@ public class Address { return false; } + public String getGeoRegion() { + return geoRegion; + } + + public void setGeoRegion(String geoRegion) { + this.geoRegion = geoRegion; + } + @Override public String toString() { return "Address{" + @@ -809,6 +819,14 @@ public class Address { '}'; } + public String getPolygonSeaweedId() { + return polygonSeaweedId; + } + + public void setPolygonSeaweedId(String polygonSeaweedId) { + this.polygonSeaweedId = polygonSeaweedId; + } + public String toMd5Hex(){ String keyProperties = "Address{" + "city='" + city + '\'' + diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/AddressProcessors.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/AddressProcessors.java index aa72e6d01..d1a15d156 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/AddressProcessors.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/AddressProcessors.java @@ -20,18 +20,16 @@ public class AddressProcessors{ static { //修改顺序需要调整process i 的值 - addressProcessors.add(new TokenProcessor()); addressProcessors.add(new ProximalCharactersProcessor()); addressProcessors.add(new PreNoiseProcessor()); addressProcessors.add(new AddressFormatProcessor()); - addressProcessors.add(new TokenProcessor()); addressProcessors.add(new FloorProcessor()); addressProcessors.add(new TooltipProcessor()); addressProcessors.add(new ProvinceProcessor()); addressProcessors.add(new CityProcessor()); addressProcessors.add(new DistrictProcessor()); - addressProcessors.add(new LoopLineProcessor()); addressProcessors.add(new TownshipProcessor()); + addressProcessors.add(new LoopLineProcessor()); addressProcessors.add(new StageAndPartitionProcessor()); addressProcessors.add(new DirectionProcessor()); addressProcessors.add(new OffsetProcessor()); @@ -44,7 +42,7 @@ public class AddressProcessors{ addressProcessors.add(new NumericProcessor()); addressProcessors.add(new ProvinceAdjustProcessor()); addressProcessors.add(new FormatProcessor()); -// addressProcessors.add(new GeoProcessor()); + addressProcessors.add(new GeoProcessor()); } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java index 6f88c20ef..fce876ec2 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java @@ -52,8 +52,9 @@ public class CityProcessor extends AbstractAddressProcessor { } if(StringUtils.isNotBlank(city) && CityStatus.SURE != address.getCityStatus() ){ String formatCity = RegionData.formatCity(city); - String possibleCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) ? parseCityFromAddress(orgAddress, address) : null; - String confirmCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) ? null : formatCity; + address.setCity(formatCity); + String possibleCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) || address.getAddressParserGoal() == null ? parseCityFromAddress(orgAddress, address) : null; + String confirmCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) || address.getAddressParserGoal() == null? null : formatCity; /**如果省份和城市信息相互矛盾 则用区域推断城市*/ if(StringUtils.isNotBlank(address.getProvince()) && !ProvinceData.belongTo(formatCity, address.getProvince())){ List possibleCities = speculateCityNameByRegion(address, city); @@ -191,15 +192,12 @@ public class CityProcessor extends AbstractAddressProcessor { }else{ weight.score += 2; } - }else if(address.contains(city)){ index = address.indexOf(city); weight.index = index; index += city.length(); if(province != null && (orgAddress.contains(province + city) || orgAddress.contains(ProvinceData.getDisplay(province) + city))){ weight.score += 2; - }else{ - weight.score += 1; } } Map> regionMap = RegionData.getAllRegions(city); @@ -208,9 +206,11 @@ public class CityProcessor extends AbstractAddressProcessor { if(matchResult !=null){ address = address.replace(matchResult.matchContent, ""); } - if(address.indexOf(region, index) > -1){ + if(address.indexOf(region + "区", index) > -1 || address.indexOf(region + "县", index) > -1){ if(province != null && (orgAddress.contains(province + region) || orgAddress.contains(ProvinceData.getDisplay(province) + region))){ weight.score += 2; + }else if(city.equals(addr.getCity())){ + weight.score += 10; }else{ weight.score += 1; } @@ -221,9 +221,13 @@ public class CityProcessor extends AbstractAddressProcessor { List alias = regionMap.get(region); if(alias != null){ for(String alia : alias){ - if(StringUtils.isNotBlank(alia) && !alia.equals(region) && address.indexOf(alia, index) > -1 + if(StringUtils.isNotBlank(alia) && !alia.equals(region) && alia.length() > 2 && address.indexOf(alia, index) > -1 && (alia.endsWith("市")|(alia.endsWith("县")))){ - weight.score += 1; + if(city.equals(addr.getCity())){ + weight.score += 10; + }else{ + weight.score += 1; + } weight.index = address.indexOf(alia, index); break; } @@ -237,16 +241,6 @@ public class CityProcessor extends AbstractAddressProcessor { } List weights = new ArrayList(weightMap.values()); if(weights.size() > 0){ - /*String street = getStreet(address); - if(street != null){ - Iterator iterator = weights.iterator(); - while(iterator.hasNext()){ - FragmentWeight fragmentWeight = iterator.next(); - if(match(street, ".*" + fragmentWeight.text + "[东南西北中一二三四五六七八九十百千\\d]?(路|街|大道|支路|村)")){ - iterator.remove(); - } - } - }*/ Collections.sort(weights); } if(weights.size() > 0){ diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/DistrictProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/DistrictProcessor.java index 5cfba96b5..697e69f8d 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/DistrictProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/DistrictProcessor.java @@ -167,6 +167,9 @@ public class DistrictProcessor extends AbstractAddressProcessor { address.setDevelopmentZone(dz); } } + if(address.getSuggestDistrict() != null && address.getDistrict() == null){ + address.setDistrict(address.getSuggestDistrict()); + } address.setAddress(orgAddress); address.setEffectiveAddress(address.getAddress()); return RepeatStatus.NEXT; diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java index 83d74a0d4..e3ef9a3da 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java @@ -5,8 +5,12 @@ import com.fangjia.core.util.HttpRequest; import com.fangjia.service.address.mode.Address; import net.sf.json.JSONArray; import net.sf.json.JSONObject; -import org.apache.commons.lang.StringUtils; -import java.util.*; +import org.apache.log4j.Logger; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; /** * 地址经纬度处理器 @@ -16,78 +20,69 @@ import java.util.*; public class GeoProcessor extends AbstractAddressProcessor { private static final String KEY = "4ccfe3c898a9489884bd2bcb1ef6dcde"; - - List POI_LEVELS = Arrays.asList("村庄", "热点商圈", "兴趣点", "门牌号", "单元号", "道路交叉路口"); - List DISTRICT_LEVEL = Arrays.asList("区县", "开发区", "乡镇", "村庄", "热点商圈", "兴趣点", "门牌号", "单元号", "道路交叉路口"); + private static final Logger logger = Logger.getLogger(GeoProcessor.class); + List POI_LEVELS = Arrays.asList("村庄", "热点商圈", "兴趣点", "门牌号", "单元号", "道路交叉路口", "门牌号"); @Override public RepeatStatus process(Address address) { - if (address.hasCoordinate()) { return RepeatStatus.NEXT; } - List queryTypes = new ArrayList(); - List addressList = new ArrayList(); - if (address.isStandardAddress()) { - addressList.add(address.getStandardAddress()); - queryTypes.add("StandardAddress"); - } - if (StringUtils.isNotBlank(address.getOrgKeywords())) { - addressList.add(address.getOrgKeywords()); - queryTypes.add("Keywords"); + String addr = address.getEffectiveAddress(); + if(addr == null || addr.length() < 3){ + addr = address.getOrgAddress(); } HttpRequest request = new HttpRequest(); Map params = new HashMap(); - params.put("address", StringUtils.join(addressList, "|")); + params.put("address", addr); params.put("key", KEY); params.put("city", address.getCity()); - params.put("batch", "true"); params.put("output", "json"); try { JSONObject response = request.getJsonContent("https://restapi.amap.com/v3/geocode/geo", params, "UTF-8"); + logger.info("高德API:" + response.toString()); if (response.getInt("status") == 1) { JSONArray geocodes = response.getJSONArray("geocodes"); - if(queryTypes.contains("StandardAddress")){ - JSONObject geocode = geocodes.getJSONObject(queryTypes.indexOf("StandardAddress")); - String level = geocode.getString("level"); - if(DISTRICT_LEVEL.contains(level)){ - if(StringUtils.isBlank(address.getDistrict())){ - address.setDistrict(RegionData.formatRegion(address.getCity(), geocode.getString("district"))); + if(geocodes.size() > 0){ + JSONObject json = geocodes.getJSONObject(0); + if(json.get("district") instanceof String){ + String city = address.getCity(); + if(city == null){ + city = RegionData.formatCity(json.getString("city")); } + String region = RegionData.formatRegion(city, json.getString("district")); + address.setGeoRegion(region); } - if(geocode.containsKey("street") && geocode.containsKey("number")){ - String geocodeAddress = geocode.getString("street") + geocode.getString("number"); - if(address.getStandardAddress().contains(geocodeAddress)){ - String location = geocode.getString("location"); - address.setLng(Double.parseDouble(location.split(",")[0])); - address.setLat(Double.parseDouble(location.split(",")[1])); - } - } - } - if(queryTypes.contains("Keywords")){ - JSONObject geocode = geocodes.getJSONObject(queryTypes.indexOf("Keywords")); - String level = geocode.getString("level"); - if(DISTRICT_LEVEL.contains(level)){ - if(StringUtils.isBlank(address.getDistrict())){ - address.setDistrict(RegionData.formatRegion(address.getCity(), geocode.getString("district"))); - } - } - if(POI_LEVELS.contains(level)){ - String formattedAddress = geocode.getString("formatted_address"); - formattedAddress = formattedAddress - .replaceFirst(geocode.getString("province"), "") - .replaceFirst(geocode.getString("city"), "") - .replaceFirst(geocode.getString("district"), ""); - if(formattedAddress.equals(address.getOrgKeywords())){ - String location = geocode.getString("location"); - address.setLng(Double.parseDouble(location.split(",")[0])); - address.setLat(Double.parseDouble(location.split(",")[1])); + if(POI_LEVELS.contains(json.getString("level"))){ + //对于门牌号级别的POI,formatted_address中包含|代表是推测出来的路号,重庆市巴南区凤林路|808号,|号楼 + if("门牌号".equals(json.getString("level"))){ + if(!(json.get("number") instanceof JSONArray) && json.getString("formatted_address").contains("|" + json.getString("number"))){ + return RepeatStatus.NEXT; + } } + String location = json.getString("location"); + address.setLng(Double.valueOf(location.split(",")[0])); + address.setLat(Double.valueOf(location.split(",")[1])); + String tmp = "{ \"_source\": [\"seaweed_id\"], \"query\": {\"geo_shape\": {\"shape\": {\"shape\": {\"type\": \"Point\",\"coordinates\":[location]},\"relation\": \"CONTAINS\"}}}}"; + tmp = tmp.replace("location", location); + Map headers = new HashMap(); + headers.put("Authorization","Basic ZGV2ZWxvcGVyOjVlOGVkZDg1MWQyZmRmYmQ3NDE1MjMyYzY3MzY3Y2Mz"); + JSONObject resp = JSONObject.fromObject(request.postWithBody("http://192.168.0.206:9200/seaweed_polygon/_search", JSONObject.fromObject(tmp),headers,"UTF-8")); + if(resp.containsKey("hits") && resp.getJSONObject("hits").containsKey("hits")){ + JSONArray hits = resp.getJSONObject("hits").getJSONArray("hits"); + if(hits.size() > 0){ + String seaweedId = hits.getJSONObject(0).getJSONObject("_source").getString("seaweed_id"); + address.setPolygonSeaweedId(seaweedId); + } + } + } } + } } catch (Exception e) { - + logger.error(e); + e.printStackTrace(); } return RepeatStatus.NEXT; } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/KeywordProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/KeywordProcessor.java index 547d8d2cb..b982d0b4f 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/KeywordProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/KeywordProcessor.java @@ -22,7 +22,7 @@ public class KeywordProcessor extends AbstractAddressProcessor { static final List KEYWORDS_PATTERNS = Arrays.asList(Pattern.compile(".*[^一二三四五六七八九十\\d]{3,}号")); static final List KEYWORDS = Arrays.asList("居委会", "辖区", "街道办", "街道", "办事处", "村委", "管理区"); static final List NOISE_PATTERNS = Arrays.asList(Pattern.compile("^.*\\d+队"), Pattern.compile("^\\d+号院$")); - static final List INVALID_KEYWORD_PATTERN = Arrays.asList(Pattern.compile("^[一二三四五六七八九十甲乙丙丁戊己庚辛A-Z\\d]+[区期]$"), Pattern.compile("碧桂园"), Pattern.compile("住宅楼"), Pattern.compile("家属院")); + static final List INVALID_KEYWORD_PATTERN = Arrays.asList(Pattern.compile("^[一二三四五六七八九十甲乙丙丁戊己庚辛A-Z\\d]+[区期]$"), Pattern.compile("碧桂园"), Pattern.compile("住宅楼"), Pattern.compile("家属[区院]"), Pattern.compile("宿舍"), Pattern.compile("生活区"), Pattern.compile("公寓")); static final Pattern INVALID_PATTERN = Pattern.compile("^([-\\\\、])+$"); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/TooltipProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/TooltipProcessor.java index 4d3b65b14..3921fddf5 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/TooltipProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/TooltipProcessor.java @@ -42,12 +42,10 @@ public class TooltipProcessor extends AbstractAddressProcessor { if(matchFirst(AddressParserPatterns.STREET, tooltip) != null){ address.setTooltip(tooltip); address.setStreet(getStreet(tooltip)); - } else if(containsDistrictName(tooltip)){ - address.setTooltip(tooltip); - } else if(RegionData.isRegion(address.getCity(), tooltip)){ + }else if(RegionData.isRegion(address.getCity(), tooltip)){ address.setSuggestDistrict(tooltip); }else{ - address.addNosie(tooltip); + address.setTooltip(tooltip); } } } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/VillageNameProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/VillageNameProcessor.java index 5236b7d55..797d9e367 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/VillageNameProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/VillageNameProcessor.java @@ -15,15 +15,6 @@ public class VillageNameProcessor extends AbstractAddressProcessor { @Override public AbstractAddressProcessor.RepeatStatus process(Address address) { String orgAddress = address.getAddress(); - if(address.getTokens()!=null){ - for(Token token:address.getTokens()){ - if("district".equals(token.getKeywordTypes())){ - address.setKeywords(token.getName()); - address.setAddress(orgAddress.replace(token.getName(),"")); - return RepeatStatus.NEXT; - } - } - } List names = tokenDistrictNames(orgAddress); List kws = new ArrayList(); for(String name : names){ diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/cnAnalyzer/CnTokenizer.java b/projects/fangjia-service/src/main/java/com/fangjia/service/cnAnalyzer/CnTokenizer.java index de010e201..c216a1eba 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/cnAnalyzer/CnTokenizer.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/cnAnalyzer/CnTokenizer.java @@ -1,24 +1,16 @@ package com.fangjia.service.cnAnalyzer; -import java.io.IOException; -import java.io.StringReader; -import java.io.UnsupportedEncodingException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import org.apache.commons.lang.StringUtils; -import org.apache.solr.client.solrj.util.ClientUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.fangjia.core.util.DataUtil; import com.goojia.solr.ganalytics.IKSegmentation; import com.goojia.solr.ganalytics.Lexeme; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.StringReader; +import java.io.UnsupportedEncodingException; +import java.util.*; /** * 房价网分词器 diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoServiceImpl.java index 3965c9f00..c38f74a88 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoServiceImpl.java @@ -240,7 +240,7 @@ public class AddressRepoServiceImpl implements AddressRepoService { } private boolean isAvailablePoi(AddressMatchMetaData data){ - if(data.getType() != null && data.getType().contains("路")){ + if(data.getType() != null && (data.getType().contains("路") || data.getType().contains("街道"))){ return false; } return true; diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/open/CommunityBasePriceService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/open/CommunityBasePriceService.java new file mode 100644 index 000000000..bc8c159de --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/open/CommunityBasePriceService.java @@ -0,0 +1,9 @@ +package com.fangjia.service.open; + +import com.fangjia.dal.mongo.open.model.CommunityBasePrice; +import com.fangjia.dal.mongo.open.model.CommunityPoi; + +public interface CommunityBasePriceService { + + CommunityBasePrice getById(String id); +} diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/open/CommunityPoiService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/open/CommunityPoiService.java new file mode 100644 index 000000000..c0eb458f1 --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/open/CommunityPoiService.java @@ -0,0 +1,8 @@ +package com.fangjia.service.open; + +import com.fangjia.dal.mongo.open.model.CommunityPoi; + +public interface CommunityPoiService { + + CommunityPoi getById(String id); +} diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/open/impl/CommunityBasePriceServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/open/impl/CommunityBasePriceServiceImpl.java new file mode 100644 index 000000000..049e5ae16 --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/open/impl/CommunityBasePriceServiceImpl.java @@ -0,0 +1,22 @@ +package com.fangjia.service.open.impl; + +import com.fangjia.dal.mongo.open.dao.CommunityBasePriceDao; +import com.fangjia.dal.mongo.open.dao.CommunityPoiDao; +import com.fangjia.dal.mongo.open.model.CommunityBasePrice; +import com.fangjia.dal.mongo.open.model.CommunityPoi; +import com.fangjia.service.open.CommunityBasePriceService; +import com.fangjia.service.open.CommunityPoiService; + +public class CommunityBasePriceServiceImpl implements CommunityBasePriceService { + + private CommunityBasePriceDao communityBasePriceDao; + + @Override + public CommunityBasePrice getById(String id) { + return communityBasePriceDao.findById(id); + } + + public void setCommunityBasePriceDao(CommunityBasePriceDao communityBasePriceDao) { + this.communityBasePriceDao = communityBasePriceDao; + } +} diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/open/impl/CommunityPoiServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/open/impl/CommunityPoiServiceImpl.java new file mode 100644 index 000000000..8ea0b05e1 --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/open/impl/CommunityPoiServiceImpl.java @@ -0,0 +1,19 @@ +package com.fangjia.service.open.impl; + +import com.fangjia.dal.mongo.open.dao.CommunityPoiDao; +import com.fangjia.dal.mongo.open.model.CommunityPoi; +import com.fangjia.service.open.CommunityPoiService; + +public class CommunityPoiServiceImpl implements CommunityPoiService { + + private CommunityPoiDao communityPoiDao; + + @Override + public CommunityPoi getById(String id) { + return communityPoiDao.findById(id); + } + + public void setCommunityPoiDao(CommunityPoiDao communityPoiDao) { + this.communityPoiDao = communityPoiDao; + } +} diff --git a/projects/fangjia-static/src/main/webapp/style/fangjia/v5/company.css b/projects/fangjia-static/src/main/webapp/style/fangjia/v5/company.css index 0b498e441..805853bc9 100644 --- a/projects/fangjia-static/src/main/webapp/style/fangjia/v5/company.css +++ b/projects/fangjia-static/src/main/webapp/style/fangjia/v5/company.css @@ -20,7 +20,7 @@ .officeMap{width:100%;height:400px;} .main_content.partner{overflow:hidden;} -.main_content.partner a{float:left;width:180px;height:120px;background:url(../../../images/fangjia/v5/partner.png) no-repeat #f0f4f7;margin-right:24px;margin-bottom:25px;cursor:pointer;} +.main_content.partner a{float:left;width:180px;height:120px;background:url(../../../images/fangjia/v5/partner.png) no-repeat #f0f4f7;margin-right:15px;margin-bottom:25px;cursor:pointer;} .partner a.partner11{background-position:0 0;} .partner a.partner12{background-position:-204px 0;}